feat: a new Minstrel restores a backup, from the web app or the command line, then rescans (#6775)

A new server (no accounts, no library) offers "Restore a backup" beside
"Create account". The file goes up as the request body to
POST /api/auth/restore with the setup token from the log in X-Setup-Token,
since the backup's admins become this server's. It loads in one transaction,
then a library scan starts and finds each file at its path. The same load runs
offline as `minstrel restore FILE`, which migrates the schema first.

That path alone gets a 1 GiB, 15-minute body allowance instead of 4 MiB and
30s, and skips the JSON-only check for cookie writes: a stale cookie from the
old server would otherwise refuse the upload, and the custom header already
rules out a cross-site form.

The library sync answers 410 cursor_ahead to a cursor past the last one the
server has handed out, read from the change sequence, so an app synced with
the old server after the backup was made starts over instead of skipping the
ids the restored server reuses. Restore now leaves each sequence as having
issued the restored maximum, so that reading holds; a compacted log still
counts its handed-out cursors.

backup.Read keeps the underlying error in its chain, so an oversized upload
answers 413 rather than "not a backup". The hosting docs say how to put the
music back first and what is entered again afterwards.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/api/api.go

@@ -99,6 +99,7 @@ func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playev
 		api.Post("/auth/login", h.handleLogin)
 		api.Post("/auth/register", h.handleRegister)
 		api.Get("/auth/setup-status", h.handleSetupStatus)
+		api.Post("/auth/restore", h.handleRestore)
 		api.Post("/auth/forgot-password", h.handleForgotPassword)
 		api.Post("/auth/reset-password", h.handleResetPassword)
 

internal/api/auth_restore.go

@@ -0,0 +1,94 @@
+package api
+
+import (
+	"context"
+	"errors"
+	"net/http"
+
+	"git.fabledsword.com/bvandeusen/minstrel/internal/apierror"
+	"git.fabledsword.com/bvandeusen/minstrel/internal/backup"
+	"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
+	"git.fabledsword.com/bvandeusen/minstrel/internal/library"
+)
+
+// SetupTokenHeader carries the setup token on a restore upload, whose body is
+// the backup file itself.
+const SetupTokenHeader = "X-Setup-Token"
+
+type restoreResp struct {
+	// Accounts is how many accounts came back; everyone signs in with the
+	// password they had.
+	Accounts int64 `json:"accounts"`
+	// ScanStarted says the library scan that refills what the backup leaves
+	// out is under way.
+	ScanStarted bool `json:"scan_started"`
+}
+
+// handleRestore implements POST /api/auth/restore: a new server, with no
+// accounts yet, loads a backup file sent as the request body. It asks for the
+// setup token from the server log, as creating the first account does, since
+// the backup's admins become this server's admins. Then it starts a library
+// scan, which finds each file at its path and fills in what the backup left
+// out.
+//
+// 200 restored · 400 not_a_backup · 403 setup_token_invalid ·
+// 409 not_empty (accounts or a library already) · 409 newer_backup
+func (h *handlers) handleRestore(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+	q := dbq.New(h.pool)
+	n, err := q.CountUsers(ctx)
+	if err != nil {
+		writeErrWithLog(w, h.logger, "restore: count users", apierror.Internal(err))
+		return
+	}
+	if n > 0 {
+		writeErr(w, apierror.Conflict("not_empty", "this server already has accounts; a backup restores into a new one"))
+		return
+	}
+	if h.requireSetupToken && !h.setupToken.Matches(r.Header.Get(SetupTokenHeader)) {
+		h.logger.Warn("restore: needs the setup token", "setup_token", h.setupToken.Value())
+		writeErr(w, apierror.Forbidden("setup_token_invalid", "the setup token from the server log is required to restore a backup"))
+		return
+	}
+
+	manifest, contents, err := backup.Read(r.Body)
+	if err != nil {
+		var tooBig *http.MaxBytesError
+		if errors.As(err, &tooBig) {
+			writeErr(w, &apierror.Error{Status: http.StatusRequestEntityTooLarge, Code: "too_large", Message: "the file is larger than a backup can be"})
+			return
+		}
+		writeErr(w, apierror.BadRequest("not_a_backup", "that file isn't a Minstrel backup"))
+		return
+	}
+	switch err := backup.Restore(ctx, h.pool, contents); {
+	case errors.Is(err, backup.ErrNotEmpty):
+		writeErr(w, apierror.Conflict("not_empty", "this server already has accounts or a library; a backup restores into a new one"))
+		return
+	case errors.Is(err, backup.ErrNewerSchema):
+		writeErr(w, apierror.Conflict("newer_backup", "the backup was made by a newer Minstrel; update this server first"))
+		return
+	case errors.Is(err, backup.ErrNotABackup):
+		writeErr(w, apierror.BadRequest("not_a_backup", "that file isn't a Minstrel backup"))
+		return
+	case err != nil:
+		writeErrWithLog(w, h.logger, "restore: load the backup", apierror.Internal(err))
+		return
+	}
+
+	out := restoreResp{}
+	if out.Accounts, err = q.CountUsers(ctx); err != nil {
+		h.logger.Warn("restore: count restored accounts", "err", err)
+	}
+	h.logger.Info("restored a backup", "made_at", manifest.CreatedAt, "build", manifest.Build,
+		"schema", manifest.Schema, "accounts", out.Accounts)
+	if h.scanner != nil {
+		started, _, err := library.TryStartScan(context.Background(), h.pool, h.scanner, h.coverart,
+			h.logger.With("source", "restore"), h.scanCfg)
+		if err != nil {
+			h.logger.Warn("restore: start the library scan", "err", err)
+		}
+		out.ScanStarted = started
+	}
+	writeJSON(w, http.StatusOK, out)
+}

internal/api/auth_restore_test.go

@@ -0,0 +1,87 @@
+package api
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"git.fabledsword.com/bvandeusen/minstrel/internal/auth"
+	"git.fabledsword.com/bvandeusen/minstrel/internal/backup"
+	"git.fabledsword.com/bvandeusen/minstrel/internal/dbtest"
+)
+
+// A new server takes a backup only with the setup token from its log, only
+// while it has no accounts, and only a file that is a backup; then the
+// backup's accounts sign in again.
+func TestRestore_ANewServerTakesABackupWithTheSetupToken(t *testing.T) {
+	h, pool := testHandlers(t)
+	ctx := context.Background()
+	truncateLibrary(t, pool)
+	resetUsers(t, h)
+	seedUser(t, pool, "restored", "abcd1234", true)
+	c, err := backup.Snapshot(ctx, pool)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var file bytes.Buffer
+	if _, err := backup.Write(&file, c, time.Now(), "test"); err != nil {
+		t.Fatal(err)
+	}
+	resetUsers(t, h)
+
+	token, err := auth.NewSetupToken()
+	if err != nil {
+		t.Fatal(err)
+	}
+	h.setupToken = token
+	h.requireSetupToken = true
+	restore := func(setupToken string, body []byte) *httptest.ResponseRecorder {
+		req := httptest.NewRequest(http.MethodPost, "/api/auth/restore", bytes.NewReader(body))
+		req.Header.Set("Content-Type", "application/x-tar")
+		if setupToken != "" {
+			req.Header.Set(SetupTokenHeader, setupToken)
+		}
+		rec := httptest.NewRecorder()
+		h.handleRestore(rec, req)
+		return rec
+	}
+
+	if w := restore("", file.Bytes()); w.Code != http.StatusForbidden {
+		t.Errorf("no setup token: status = %d, want 403", w.Code)
+	}
+	if w := restore("wrong", file.Bytes()); w.Code != http.StatusForbidden {
+		t.Errorf("a wrong setup token: status = %d, want 403", w.Code)
+	}
+	if w := restore(token.Value(), []byte("not a tar file")); w.Code != http.StatusBadRequest ||
+		!strings.Contains(w.Body.String(), "not_a_backup") {
+		t.Errorf("not a backup: status = %d body = %s, want 400 not_a_backup", w.Code, w.Body.String())
+	}
+	var n int
+	if err := pool.QueryRow(ctx, "SELECT count(*) FROM users").Scan(&n); err != nil || n != 0 {
+		t.Fatalf("a refused restore left %d accounts (%v)", n, err)
+	}
+
+	w := restore(token.Value(), file.Bytes())
+	if w.Code != http.StatusOK {
+		t.Fatalf("restore: status = %d body = %s", w.Code, w.Body.String())
+	}
+	var out restoreResp
+	if err := json.NewDecoder(w.Body).Decode(&out); err != nil || out.Accounts != 1 {
+		t.Errorf("restore answered %+v (%v), want one account", out, err)
+	}
+	var admin bool
+	if err := pool.QueryRow(ctx, "SELECT is_admin FROM users WHERE username = $1",
+		dbtest.TestUserPrefix+"restored").Scan(&admin); err != nil || !admin {
+		t.Errorf("the restored admin: is_admin = %v, %v", admin, err)
+	}
+
+	if w := restore(token.Value(), file.Bytes()); w.Code != http.StatusConflict ||
+		!strings.Contains(w.Body.String(), "not_empty") {
+		t.Errorf("a second restore: status = %d body = %s, want 409 not_empty", w.Code, w.Body.String())
+	}
+}

internal/api/library_sync.go

@@ -69,6 +69,23 @@ func (h *handlers) handleLibrarySync(w http.ResponseWriter, r *http.Request) {
 			})
 			return
 		}
+		// A cursor past anything this server has handed out comes from a
+		// server restored from an older backup: the changes between the
+		// backup and the cursor are gone, and the ones made from here on would
+		// reuse those ids. The app resets the same way.
+		lastCursor, err := q.GetLastIssuedLibraryChangeCursor(ctx)
+		if err != nil {
+			writeErrWithLog(w, h.logger, "library_sync: GetLastIssuedLibraryChangeCursor", err)
+			return
+		}
+		if since > lastCursor {
+			writeErr(w, &apierror.Error{
+				Status:  http.StatusGone,
+				Code:    "cursor_ahead",
+				Message: "this server holds no changes up to that cursor; reset client cursor to 0 and resync",
+			})
+			return
+		}
 	}
 
 	changes, err := q.GetLibraryChangesSince(ctx, dbq.GetLibraryChangesSinceParams{

internal/api/library_sync_test.go

@@ -6,6 +6,7 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"strconv"
+	"strings"
 	"testing"
 
 	"git.fabledsword.com/bvandeusen/minstrel/internal/auth"
@@ -204,3 +205,48 @@ func TestSplitOnce(t *testing.T) {
 		}
 	}
 }
+
+// A server restored from an older backup has handed out fewer cursors than an
+// app remembers; the app is told to reset rather than wait for ids it would
+// then skip. A compacted log is not that: the cursor the server handed out
+// still counts.
+func TestLibrarySync_ACursorPastTheServersLastChangeIsReset(t *testing.T) {
+	h, pool := testHandlers(t)
+	truncateLibrary(t, pool)
+	u := seedUser(t, pool, "alice", "x", false)
+
+	ctx := context.Background()
+	var artistID string
+	if err := pool.QueryRow(ctx,
+		`INSERT INTO artists (name, sort_name) VALUES ('A', 'A') RETURNING id::text`).Scan(&artistID); err != nil {
+		t.Fatal(err)
+	}
+	tx, err := pool.Begin(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := sync.LogChange(ctx, tx, sync.EntityArtist, artistID, sync.OpUpsert); err != nil {
+		t.Fatal(err)
+	}
+	if err := tx.Commit(ctx); err != nil {
+		t.Fatal(err)
+	}
+	var last int64
+	if err := pool.QueryRow(ctx, `SELECT max(id) FROM library_changes`).Scan(&last); err != nil {
+		t.Fatal(err)
+	}
+
+	w := callLibrarySync(h, u, strconv.FormatInt(last+50, 10))
+	if w.Code != http.StatusGone || !strings.Contains(w.Body.String(), "cursor_ahead") {
+		t.Errorf("a cursor past the last change: status = %d body = %s, want 410 cursor_ahead", w.Code, w.Body.String())
+	}
+
+	// The compactor trims every row; the app's cursor is still one this
+	// server handed out.
+	if _, err := pool.Exec(ctx, `DELETE FROM library_changes`); err != nil {
+		t.Fatal(err)
+	}
+	if w := callLibrarySync(h, u, strconv.FormatInt(last, 10)); w.Code != http.StatusNoContent {
+		t.Errorf("the last cursor after compaction: status = %d body = %s, want 204", w.Code, w.Body.String())
+	}
+}

internal/backup/archive.go

@@ -175,14 +175,14 @@ func Read(r io.Reader) (Manifest, Contents, error) {
 			break
 		}
 		if err != nil {
-			return m, c, fmt.Errorf("%w: %v", ErrNotABackup, err)
+			return m, c, fmt.Errorf("%w: %w", ErrNotABackup, err)
 		}
 		if hdr.Size > maxMember {
 			return m, c, fmt.Errorf("%w: %s is too large", ErrNotABackup, hdr.Name)
 		}
 		data, err := io.ReadAll(io.LimitReader(tr, maxMember))
 		if err != nil {
-			return m, c, fmt.Errorf("%w: %v", ErrNotABackup, err)
+			return m, c, fmt.Errorf("%w: %w", ErrNotABackup, err)
 		}
 		if first {
 			first = false

internal/backup/backup_test.go

@@ -224,10 +224,19 @@ func TestARestoredInstanceHoldsWhatTheBackupHeld_Integration(t *testing.T) {
 	}
 
 	// The change feed carries on after the restored ids.
-	var restoredMax, next int64
+	var restoredMax, issued, next int64
 	if err := pool.QueryRow(ctx, "SELECT max(id) FROM library_changes").Scan(&restoredMax); err != nil {
 		t.Fatal(err)
 	}
+	// The sequence says the restored history ends there, which is what the
+	// library sync compares an app's cursor with.
+	if err := pool.QueryRow(ctx, `SELECT COALESCE(pg_sequence_last_value(
+		pg_get_serial_sequence('library_changes', 'id')::regclass), 0)`).Scan(&issued); err != nil {
+		t.Fatal(err)
+	}
+	if issued != restoredMax {
+		t.Errorf("the change sequence stands at %d, want the restored %d", issued, restoredMax)
+	}
 	if err := pool.QueryRow(ctx, `INSERT INTO library_changes (entity_type, entity_id, op)
 		VALUES ('track', $1, 'upsert') RETURNING id`, track).Scan(&next); err != nil {
 		t.Fatal(err)

internal/backup/restore.go

@@ -158,14 +158,18 @@ func restoreTable(ctx context.Context, tx pgx.Tx, table string, rows [][]byte) e
 	}
 
 	// A sequence carries on after the highest restored value, so the next row
-	// doesn't collide with one from the backup.
+	// doesn't collide with one from the backup. It is left as having issued
+	// that value, so the sequence itself says how far the restored history
+	// goes: the library sync compares an app's cursor with it.
 	for _, c := range cols {
 		if !c.sequence {
 			continue
 		}
 		col := pgx.Identifier{c.name}.Sanitize()
-		if _, err := tx.Exec(ctx, fmt.Sprintf(
-			"SELECT setval(pg_get_serial_sequence($1, $2), COALESCE((SELECT max(%s) FROM %s), 0) + 1, false)", col, ident),
+		if _, err := tx.Exec(ctx, fmt.Sprintf(`
+			SELECT CASE WHEN m > 0 THEN setval(pg_get_serial_sequence($1, $2), m, true)
+			            ELSE setval(pg_get_serial_sequence($1, $2), 1, false) END
+			  FROM (SELECT COALESCE(max(%s), 0) AS m FROM %s) AS top`, col, ident),
 			table, c.name); err != nil {
 			return fmt.Errorf("backup: carry on %s's sequence: %w", table, err)
 		}

internal/db/dbq/library_changes.sql.go

@@ -26,6 +26,20 @@ func (q *Queries) DeleteOldLibraryChanges(ctx context.Context, changedAt pgtype.
 	return result.RowsAffected(), nil
 }
 
+const getLastIssuedLibraryChangeCursor = `-- name: GetLastIssuedLibraryChangeCursor :one
+SELECT COALESCE(pg_sequence_last_value(pg_get_serial_sequence('library_changes', 'id')::regclass), 0)::BIGINT
+`
+
+// The highest cursor this server has handed out, even once the compactor has
+// trimmed the rows: the id sequence's last value. A restored server's
+// sequence stands at the backup's last change.
+func (q *Queries) GetLastIssuedLibraryChangeCursor(ctx context.Context) (int64, error) {
+	row := q.db.QueryRow(ctx, getLastIssuedLibraryChangeCursor)
+	var column_1 int64
+	err := row.Scan(&column_1)
+	return column_1, err
+}
+
 const getLibraryChangesSince = `-- name: GetLibraryChangesSince :many
 SELECT id, entity_type, entity_id, op, changed_at
 FROM library_changes

internal/db/queries/library_changes.sql

@@ -22,6 +22,12 @@ LIMIT $2;
 -- name: GetMaxLibraryChangeCursor :one
 SELECT COALESCE(MAX(id), 0)::BIGINT FROM library_changes;
 
+-- name: GetLastIssuedLibraryChangeCursor :one
+-- The highest cursor this server has handed out, even once the compactor has
+-- trimmed the rows: the id sequence's last value. A restored server's
+-- sequence stands at the backup's last change.
+SELECT COALESCE(pg_sequence_last_value(pg_get_serial_sequence('library_changes', 'id')::regclass), 0)::BIGINT;
+
 -- name: GetMinLibraryChangeCursor :one
 SELECT COALESCE(MIN(id), 0)::BIGINT FROM library_changes;
 

internal/server/hygiene.go

@@ -18,6 +18,23 @@ const maxRequestBody = 4 << 20
 // bodyReadTimeout bounds how long a client may take to send a request body.
 const bodyReadTimeout = 30 * time.Second
 
+// restorePath takes a whole backup file, so it gets room for one: the file is
+// the database alone (the music travels separately), which is tens of MB for
+// a large library.
+const (
+	restorePath        = "/api/auth/restore"
+	maxRestoreBody     = 1 << 30
+	restoreReadTimeout = 15 * time.Minute
+)
+
+// bodyLimits is the size and time a request may take to send its body.
+func bodyLimits(r *http.Request) (int64, time.Duration) {
+	if r.Method == http.MethodPost && r.URL.Path == restorePath {
+		return maxRestoreBody, restoreReadTimeout
+	}
+	return maxRequestBody, bodyReadTimeout
+}
+
 // limitRequestBody caps the body size and the time allowed to deliver it.
 //
 // Why not http.Server.ReadTimeout: that deadline stays armed for the whole
@@ -35,8 +52,9 @@ func limitRequestBody(next http.Handler) http.Handler {
 		rc := http.NewResponseController(w)
 		// Best-effort: a ResponseWriter that can't set deadlines (a test
 		// recorder) still gets the size limit.
-		armed := rc.SetReadDeadline(time.Now().Add(bodyReadTimeout)) == nil
-		body := http.MaxBytesReader(w, r.Body, maxRequestBody)
+		size, wait := bodyLimits(r)
+		armed := rc.SetReadDeadline(time.Now().Add(wait)) == nil
+		body := http.MaxBytesReader(w, r.Body, size)
 		if armed {
 			body = &deadlineClearingBody{ReadCloser: body, rc: rc}
 		}
@@ -83,6 +101,10 @@ func (b *deadlineClearingBody) Close() error {
 // attaches on its own, so they are not checked. The Android app does send
 // the cookie, but every body it sends is JSON (Retrofit's kotlinx converter)
 // and its bodiless writes carry no Content-Type, so it passes unchanged.
+//
+// The restore upload is a backup file, not JSON. It is safe without the check:
+// it acts only with the setup token in a request header, which a form cannot
+// send and a cross-site fetch cannot send without a preflight.
 func requireJSONForCookieWrites(next http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		switch r.Method {
@@ -90,7 +112,7 @@ func requireJSONForCookieWrites(next http.Handler) http.Handler {
 			next.ServeHTTP(w, r)
 			return
 		}
-		if !strings.HasPrefix(r.URL.Path, "/api/") {
+		if !strings.HasPrefix(r.URL.Path, "/api/") || r.URL.Path == restorePath {
 			next.ServeHTTP(w, r)
 			return
 		}

internal/server/hygiene_test.go

@@ -40,6 +40,21 @@ func TestLimitRequestBody_RejectsOversizedBody(t *testing.T) {
 	}
 }
 
+// A backup file is far larger than any other body, and only the restore
+// upload may send one.
+func TestLimitRequestBody_TheRestoreUploadTakesABackupFile(t *testing.T) {
+	h := limitRequestBody(okHandler())
+	file := strings.NewReader(strings.Repeat("x", maxRequestBody*2))
+	w := httptest.NewRecorder()
+	h.ServeHTTP(w, httptest.NewRequest(http.MethodPost, restorePath, file))
+	if w.Code != http.StatusNoContent {
+		t.Errorf("a backup-sized restore upload: status = %d, want 204", w.Code)
+	}
+	if size, wait := bodyLimits(httptest.NewRequest(http.MethodPost, "/api/playlists", nil)); size != maxRequestBody || wait != bodyReadTimeout {
+		t.Errorf("another path's limits = %d, %v; want the ordinary ones", size, wait)
+	}
+}
+
 // The cases a cross-site form or no-preflight fetch could produce must be
 // refused when the session cookie rides along; everything a real client
 // sends must pass.
@@ -55,6 +70,7 @@ func TestRequireJSONForCookieWrites(t *testing.T) {
 		want        int
 	}{
 		{"form post with cookie", http.MethodPost, "/api/me/password", "application/x-www-form-urlencoded", "a=b", true, http.StatusUnsupportedMediaType},
+		{"restore upload with a stale cookie", http.MethodPost, "/api/auth/restore", "application/x-tar", "tar", true, http.StatusNoContent},
 		{"text/plain post with cookie", http.MethodPost, "/api/me/password", "text/plain", `{"a":1}`, true, http.StatusUnsupportedMediaType},
 		{"multipart with cookie", http.MethodPut, "/api/me/profile", "multipart/form-data; boundary=x", "--x--", true, http.StatusUnsupportedMediaType},
 		{"body without content type", http.MethodPost, "/api/me/profile", "", `{"a":1}`, true, http.StatusUnsupportedMediaType},