From 29042e4eb2cd380137702b7bea4fac1a316895a8 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Mon, 2 Nov 2026 16:10:06 -0500 Subject: [PATCH] feat: a new Minstrel restores a backup, from the web app or the command line, then rescans (#6775) A new server (no accounts, no library) offers "Restore a backup" beside "Create account". The file goes up as the request body to POST /api/auth/restore with the setup token from the log in X-Setup-Token, since the backup's admins become this server's. It loads in one transaction, then a library scan starts and finds each file at its path. The same load runs offline as `minstrel restore FILE`, which migrates the schema first. That path alone gets a 1 GiB, 15-minute body allowance instead of 4 MiB and 30s, and skips the JSON-only check for cookie writes: a stale cookie from the old server would otherwise refuse the upload, and the custom header already rules out a cross-site form. The library sync answers 410 cursor_ahead to a cursor past the last one the server has handed out, read from the change sequence, so an app synced with the old server after the backup was made starts over instead of skipping the ids the restored server reuses. Restore now leaves each sequence as having issued the restored maximum, so that reading holds; a compacted log still counts its handed-out cursors. backup.Read keeps the underlying error in its chain, so an oversized upload answers 413 rather than "not a backup". The hosting docs say how to put the music back first and what is entered again afterwards. Co-Authored-By: Claude Opus 5.5 --- cmd/minstrel/main.go | 6 ++ cmd/minstrel/restore.go | 76 +++++++++++++++ docs/hosting.md | 28 ++++++ internal/api/api.go | 1 + internal/api/auth_restore.go | 94 +++++++++++++++++++ internal/api/auth_restore_test.go | 87 +++++++++++++++++ internal/api/library_sync.go | 17 ++++ internal/api/library_sync_test.go | 46 +++++++++ internal/backup/archive.go | 4 +- internal/backup/backup_test.go | 11 ++- internal/backup/restore.go | 10 +- internal/db/dbq/library_changes.sql.go | 14 +++ internal/db/queries/library_changes.sql | 6 ++ internal/server/hygiene.go | 28 +++++- internal/server/hygiene_test.go | 16 ++++ web/src/lib/auth/store.svelte.ts | 16 +++- .../lib/components/auth/RestoreForm.svelte | 94 +++++++++++++++++++ web/src/routes/register/+page.svelte | 23 ++++- web/src/routes/register/register.test.ts | 43 ++++++++- 19 files changed, 607 insertions(+), 13 deletions(-) create mode 100644 cmd/minstrel/restore.go create mode 100644 internal/api/auth_restore.go create mode 100644 internal/api/auth_restore_test.go create mode 100644 web/src/lib/components/auth/RestoreForm.svelte diff --git a/cmd/minstrel/main.go b/cmd/minstrel/main.go index 280d1815..27103396 100644 --- a/cmd/minstrel/main.go +++ b/cmd/minstrel/main.go @@ -61,6 +61,12 @@ func main() { os.Exit(1) } return + case "restore": + if err := runRestore(os.Args[2:]); err != nil { + fmt.Fprintf(os.Stderr, "minstrel restore: %v\n", err) + os.Exit(1) + } + return } } if err := run(); err != nil { diff --git a/cmd/minstrel/restore.go b/cmd/minstrel/restore.go new file mode 100644 index 00000000..cbe214e1 --- /dev/null +++ b/cmd/minstrel/restore.go @@ -0,0 +1,76 @@ +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "io" + "os" + + "git.fabledsword.com/bvandeusen/minstrel/internal/backup" + "git.fabledsword.com/bvandeusen/minstrel/internal/config" + "git.fabledsword.com/bvandeusen/minstrel/internal/db" + "git.fabledsword.com/bvandeusen/minstrel/internal/logging" +) + +const restoreUsage = "usage: minstrel restore [-config PATH] FILE" + +// runRestore loads a backup file into a new Minstrel's database: one with no +// accounts and no library yet. It brings the schema up to date first, so a +// backup from an older Minstrel loads into this one. The server isn't running +// meanwhile, so the library scan waits for it to start. +func runRestore(args []string) error { + fs := flag.NewFlagSet("restore", flag.ContinueOnError) + configPath := fs.String("config", os.Getenv("MINSTREL_CONFIG"), "path to YAML config file") + if err := fs.Parse(args); err != nil { + return err + } + if fs.NArg() != 1 { + return errors.New(restoreUsage) + } + cfg, err := config.Load(*configPath) + if err != nil { + return fmt.Errorf("load config: %w", err) + } + if cfg.Database.URL == "" { + return errors.New("no database URL configured (set it in the config file or MINSTREL_DATABASE_URL)") + } + logger, err := logging.New(os.Stderr, cfg.Log.Level, cfg.Log.Format) + if err != nil { + return fmt.Errorf("init logger: %w", err) + } + + f, err := os.Open(fs.Arg(0)) + if err != nil { + return err + } + defer func() { _ = f.Close() }() + manifest, contents, err := backup.Read(f) + if err != nil { + return err + } + if err := db.Migrate(cfg.Database.URL, logger); err != nil { + return fmt.Errorf("migrate: %w", err) + } + ctx := context.Background() + pool, err := db.Open(ctx, cfg.Database.URL) + if err != nil { + return fmt.Errorf("open db: %w", err) + } + defer pool.Close() + if err := backup.Restore(ctx, pool, contents); err != nil { + return err + } + printRestored(os.Stdout, manifest) + return nil +} + +// printRestored says what came back and what happens next. +func printRestored(w io.Writer, m backup.Manifest) { + _, _ = fmt.Fprintf(w, "minstrel: restored the backup made %s by Minstrel %s (%d accounts, %d tracks, %d playlists).\n", + m.CreatedAt.Local().Format("2 Jan 2006 15:04 MST"), m.Build, + m.Rows["users"], m.Rows["tracks"], m.Rows["playlists"]) + _, _ = fmt.Fprintln(w, "Start the server. The library scan (at startup, or Scan now under Admin › Library) finds each music file at its path, and the artwork, tags and analysis the backup leaves out are fetched and worked out again.") + _, _ = fmt.Fprintln(w, "Everyone signs in with the password they had. Keys for other services, and the backup's own destination, are entered again in Admin.") +} diff --git a/docs/hosting.md b/docs/hosting.md index 28207eee..20e25e65 100644 --- a/docs/hosting.md +++ b/docs/hosting.md @@ -117,6 +117,34 @@ refusals in a row stop that night's music, since that is usually a key that may not write under `music/`. A bucket's upload limit applies to the music as well. +### Restoring + +A backup restores into a new Minstrel: an empty database, with no accounts +and no library yet. + +1. **Put the music back first**, at the path the library had. The restored + settings name the old library folders, and each track is found again by + its path. If the backup includes the music, copy what is inside `music/` + into the library folder (with several library folders, into the folder + that holds them all). +2. **Load the backup**, either way: + - In the web app, choose **Create account**, then **Restore a backup**. + Pick a `minstrel-….tar` file from the backup's `database/` folder and + enter the setup token from the log (see *First account*). + - Or, with the server stopped, run + `docker compose run --rm minstrel restore /backups/database/minstrel-20261102T030000Z.tar`, + with the file on a volume the container can read. It brings the database + schema up to date first, so a backup from an older Minstrel loads into a + newer one; a backup from a newer Minstrel is refused. +3. **Sign in** with the password you had. Everyone else's accounts, likes, + playlists and history are back too. + +The library scan then finds each file, and the artwork, tags, fingerprints +and loudness are fetched or worked out again over the following hours. Keys +for other services (ListenBrainz, AcoustID, artwork and tag providers, +email, download clients) and the backup's own destination are entered again +under **Admin**. An app that synced with the old server starts its sync over. + ## Android app The app connects over whatever URL you give it, HTTP or HTTPS. Once the diff --git a/internal/api/api.go b/internal/api/api.go index d0905edb..e807c05a 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -99,6 +99,7 @@ func Mount(r chi.Router, pool *pgxpool.Pool, logger *slog.Logger, events *playev api.Post("/auth/login", h.handleLogin) api.Post("/auth/register", h.handleRegister) api.Get("/auth/setup-status", h.handleSetupStatus) + api.Post("/auth/restore", h.handleRestore) api.Post("/auth/forgot-password", h.handleForgotPassword) api.Post("/auth/reset-password", h.handleResetPassword) diff --git a/internal/api/auth_restore.go b/internal/api/auth_restore.go new file mode 100644 index 00000000..8a8e39a2 --- /dev/null +++ b/internal/api/auth_restore.go @@ -0,0 +1,94 @@ +package api + +import ( + "context" + "errors" + "net/http" + + "git.fabledsword.com/bvandeusen/minstrel/internal/apierror" + "git.fabledsword.com/bvandeusen/minstrel/internal/backup" + "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" + "git.fabledsword.com/bvandeusen/minstrel/internal/library" +) + +// SetupTokenHeader carries the setup token on a restore upload, whose body is +// the backup file itself. +const SetupTokenHeader = "X-Setup-Token" + +type restoreResp struct { + // Accounts is how many accounts came back; everyone signs in with the + // password they had. + Accounts int64 `json:"accounts"` + // ScanStarted says the library scan that refills what the backup leaves + // out is under way. + ScanStarted bool `json:"scan_started"` +} + +// handleRestore implements POST /api/auth/restore: a new server, with no +// accounts yet, loads a backup file sent as the request body. It asks for the +// setup token from the server log, as creating the first account does, since +// the backup's admins become this server's admins. Then it starts a library +// scan, which finds each file at its path and fills in what the backup left +// out. +// +// 200 restored · 400 not_a_backup · 403 setup_token_invalid · +// 409 not_empty (accounts or a library already) · 409 newer_backup +func (h *handlers) handleRestore(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + q := dbq.New(h.pool) + n, err := q.CountUsers(ctx) + if err != nil { + writeErrWithLog(w, h.logger, "restore: count users", apierror.Internal(err)) + return + } + if n > 0 { + writeErr(w, apierror.Conflict("not_empty", "this server already has accounts; a backup restores into a new one")) + return + } + if h.requireSetupToken && !h.setupToken.Matches(r.Header.Get(SetupTokenHeader)) { + h.logger.Warn("restore: needs the setup token", "setup_token", h.setupToken.Value()) + writeErr(w, apierror.Forbidden("setup_token_invalid", "the setup token from the server log is required to restore a backup")) + return + } + + manifest, contents, err := backup.Read(r.Body) + if err != nil { + var tooBig *http.MaxBytesError + if errors.As(err, &tooBig) { + writeErr(w, &apierror.Error{Status: http.StatusRequestEntityTooLarge, Code: "too_large", Message: "the file is larger than a backup can be"}) + return + } + writeErr(w, apierror.BadRequest("not_a_backup", "that file isn't a Minstrel backup")) + return + } + switch err := backup.Restore(ctx, h.pool, contents); { + case errors.Is(err, backup.ErrNotEmpty): + writeErr(w, apierror.Conflict("not_empty", "this server already has accounts or a library; a backup restores into a new one")) + return + case errors.Is(err, backup.ErrNewerSchema): + writeErr(w, apierror.Conflict("newer_backup", "the backup was made by a newer Minstrel; update this server first")) + return + case errors.Is(err, backup.ErrNotABackup): + writeErr(w, apierror.BadRequest("not_a_backup", "that file isn't a Minstrel backup")) + return + case err != nil: + writeErrWithLog(w, h.logger, "restore: load the backup", apierror.Internal(err)) + return + } + + out := restoreResp{} + if out.Accounts, err = q.CountUsers(ctx); err != nil { + h.logger.Warn("restore: count restored accounts", "err", err) + } + h.logger.Info("restored a backup", "made_at", manifest.CreatedAt, "build", manifest.Build, + "schema", manifest.Schema, "accounts", out.Accounts) + if h.scanner != nil { + started, _, err := library.TryStartScan(context.Background(), h.pool, h.scanner, h.coverart, + h.logger.With("source", "restore"), h.scanCfg) + if err != nil { + h.logger.Warn("restore: start the library scan", "err", err) + } + out.ScanStarted = started + } + writeJSON(w, http.StatusOK, out) +} diff --git a/internal/api/auth_restore_test.go b/internal/api/auth_restore_test.go new file mode 100644 index 00000000..fa23be36 --- /dev/null +++ b/internal/api/auth_restore_test.go @@ -0,0 +1,87 @@ +package api + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "git.fabledsword.com/bvandeusen/minstrel/internal/auth" + "git.fabledsword.com/bvandeusen/minstrel/internal/backup" + "git.fabledsword.com/bvandeusen/minstrel/internal/dbtest" +) + +// A new server takes a backup only with the setup token from its log, only +// while it has no accounts, and only a file that is a backup; then the +// backup's accounts sign in again. +func TestRestore_ANewServerTakesABackupWithTheSetupToken(t *testing.T) { + h, pool := testHandlers(t) + ctx := context.Background() + truncateLibrary(t, pool) + resetUsers(t, h) + seedUser(t, pool, "restored", "abcd1234", true) + c, err := backup.Snapshot(ctx, pool) + if err != nil { + t.Fatal(err) + } + var file bytes.Buffer + if _, err := backup.Write(&file, c, time.Now(), "test"); err != nil { + t.Fatal(err) + } + resetUsers(t, h) + + token, err := auth.NewSetupToken() + if err != nil { + t.Fatal(err) + } + h.setupToken = token + h.requireSetupToken = true + restore := func(setupToken string, body []byte) *httptest.ResponseRecorder { + req := httptest.NewRequest(http.MethodPost, "/api/auth/restore", bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/x-tar") + if setupToken != "" { + req.Header.Set(SetupTokenHeader, setupToken) + } + rec := httptest.NewRecorder() + h.handleRestore(rec, req) + return rec + } + + if w := restore("", file.Bytes()); w.Code != http.StatusForbidden { + t.Errorf("no setup token: status = %d, want 403", w.Code) + } + if w := restore("wrong", file.Bytes()); w.Code != http.StatusForbidden { + t.Errorf("a wrong setup token: status = %d, want 403", w.Code) + } + if w := restore(token.Value(), []byte("not a tar file")); w.Code != http.StatusBadRequest || + !strings.Contains(w.Body.String(), "not_a_backup") { + t.Errorf("not a backup: status = %d body = %s, want 400 not_a_backup", w.Code, w.Body.String()) + } + var n int + if err := pool.QueryRow(ctx, "SELECT count(*) FROM users").Scan(&n); err != nil || n != 0 { + t.Fatalf("a refused restore left %d accounts (%v)", n, err) + } + + w := restore(token.Value(), file.Bytes()) + if w.Code != http.StatusOK { + t.Fatalf("restore: status = %d body = %s", w.Code, w.Body.String()) + } + var out restoreResp + if err := json.NewDecoder(w.Body).Decode(&out); err != nil || out.Accounts != 1 { + t.Errorf("restore answered %+v (%v), want one account", out, err) + } + var admin bool + if err := pool.QueryRow(ctx, "SELECT is_admin FROM users WHERE username = $1", + dbtest.TestUserPrefix+"restored").Scan(&admin); err != nil || !admin { + t.Errorf("the restored admin: is_admin = %v, %v", admin, err) + } + + if w := restore(token.Value(), file.Bytes()); w.Code != http.StatusConflict || + !strings.Contains(w.Body.String(), "not_empty") { + t.Errorf("a second restore: status = %d body = %s, want 409 not_empty", w.Code, w.Body.String()) + } +} diff --git a/internal/api/library_sync.go b/internal/api/library_sync.go index 932d7b93..ecd45de2 100644 --- a/internal/api/library_sync.go +++ b/internal/api/library_sync.go @@ -69,6 +69,23 @@ func (h *handlers) handleLibrarySync(w http.ResponseWriter, r *http.Request) { }) return } + // A cursor past anything this server has handed out comes from a + // server restored from an older backup: the changes between the + // backup and the cursor are gone, and the ones made from here on would + // reuse those ids. The app resets the same way. + lastCursor, err := q.GetLastIssuedLibraryChangeCursor(ctx) + if err != nil { + writeErrWithLog(w, h.logger, "library_sync: GetLastIssuedLibraryChangeCursor", err) + return + } + if since > lastCursor { + writeErr(w, &apierror.Error{ + Status: http.StatusGone, + Code: "cursor_ahead", + Message: "this server holds no changes up to that cursor; reset client cursor to 0 and resync", + }) + return + } } changes, err := q.GetLibraryChangesSince(ctx, dbq.GetLibraryChangesSinceParams{ diff --git a/internal/api/library_sync_test.go b/internal/api/library_sync_test.go index 4539a4a4..84aaf277 100644 --- a/internal/api/library_sync_test.go +++ b/internal/api/library_sync_test.go @@ -6,6 +6,7 @@ import ( "net/http" "net/http/httptest" "strconv" + "strings" "testing" "git.fabledsword.com/bvandeusen/minstrel/internal/auth" @@ -204,3 +205,48 @@ func TestSplitOnce(t *testing.T) { } } } + +// A server restored from an older backup has handed out fewer cursors than an +// app remembers; the app is told to reset rather than wait for ids it would +// then skip. A compacted log is not that: the cursor the server handed out +// still counts. +func TestLibrarySync_ACursorPastTheServersLastChangeIsReset(t *testing.T) { + h, pool := testHandlers(t) + truncateLibrary(t, pool) + u := seedUser(t, pool, "alice", "x", false) + + ctx := context.Background() + var artistID string + if err := pool.QueryRow(ctx, + `INSERT INTO artists (name, sort_name) VALUES ('A', 'A') RETURNING id::text`).Scan(&artistID); err != nil { + t.Fatal(err) + } + tx, err := pool.Begin(ctx) + if err != nil { + t.Fatal(err) + } + if err := sync.LogChange(ctx, tx, sync.EntityArtist, artistID, sync.OpUpsert); err != nil { + t.Fatal(err) + } + if err := tx.Commit(ctx); err != nil { + t.Fatal(err) + } + var last int64 + if err := pool.QueryRow(ctx, `SELECT max(id) FROM library_changes`).Scan(&last); err != nil { + t.Fatal(err) + } + + w := callLibrarySync(h, u, strconv.FormatInt(last+50, 10)) + if w.Code != http.StatusGone || !strings.Contains(w.Body.String(), "cursor_ahead") { + t.Errorf("a cursor past the last change: status = %d body = %s, want 410 cursor_ahead", w.Code, w.Body.String()) + } + + // The compactor trims every row; the app's cursor is still one this + // server handed out. + if _, err := pool.Exec(ctx, `DELETE FROM library_changes`); err != nil { + t.Fatal(err) + } + if w := callLibrarySync(h, u, strconv.FormatInt(last, 10)); w.Code != http.StatusNoContent { + t.Errorf("the last cursor after compaction: status = %d body = %s, want 204", w.Code, w.Body.String()) + } +} diff --git a/internal/backup/archive.go b/internal/backup/archive.go index 895ff506..2c725175 100644 --- a/internal/backup/archive.go +++ b/internal/backup/archive.go @@ -175,14 +175,14 @@ func Read(r io.Reader) (Manifest, Contents, error) { break } if err != nil { - return m, c, fmt.Errorf("%w: %v", ErrNotABackup, err) + return m, c, fmt.Errorf("%w: %w", ErrNotABackup, err) } if hdr.Size > maxMember { return m, c, fmt.Errorf("%w: %s is too large", ErrNotABackup, hdr.Name) } data, err := io.ReadAll(io.LimitReader(tr, maxMember)) if err != nil { - return m, c, fmt.Errorf("%w: %v", ErrNotABackup, err) + return m, c, fmt.Errorf("%w: %w", ErrNotABackup, err) } if first { first = false diff --git a/internal/backup/backup_test.go b/internal/backup/backup_test.go index 2043553c..fbb9164f 100644 --- a/internal/backup/backup_test.go +++ b/internal/backup/backup_test.go @@ -224,10 +224,19 @@ func TestARestoredInstanceHoldsWhatTheBackupHeld_Integration(t *testing.T) { } // The change feed carries on after the restored ids. - var restoredMax, next int64 + var restoredMax, issued, next int64 if err := pool.QueryRow(ctx, "SELECT max(id) FROM library_changes").Scan(&restoredMax); err != nil { t.Fatal(err) } + // The sequence says the restored history ends there, which is what the + // library sync compares an app's cursor with. + if err := pool.QueryRow(ctx, `SELECT COALESCE(pg_sequence_last_value( + pg_get_serial_sequence('library_changes', 'id')::regclass), 0)`).Scan(&issued); err != nil { + t.Fatal(err) + } + if issued != restoredMax { + t.Errorf("the change sequence stands at %d, want the restored %d", issued, restoredMax) + } if err := pool.QueryRow(ctx, `INSERT INTO library_changes (entity_type, entity_id, op) VALUES ('track', $1, 'upsert') RETURNING id`, track).Scan(&next); err != nil { t.Fatal(err) diff --git a/internal/backup/restore.go b/internal/backup/restore.go index 2020b7fa..31d191dd 100644 --- a/internal/backup/restore.go +++ b/internal/backup/restore.go @@ -158,14 +158,18 @@ func restoreTable(ctx context.Context, tx pgx.Tx, table string, rows [][]byte) e } // A sequence carries on after the highest restored value, so the next row - // doesn't collide with one from the backup. + // doesn't collide with one from the backup. It is left as having issued + // that value, so the sequence itself says how far the restored history + // goes: the library sync compares an app's cursor with it. for _, c := range cols { if !c.sequence { continue } col := pgx.Identifier{c.name}.Sanitize() - if _, err := tx.Exec(ctx, fmt.Sprintf( - "SELECT setval(pg_get_serial_sequence($1, $2), COALESCE((SELECT max(%s) FROM %s), 0) + 1, false)", col, ident), + if _, err := tx.Exec(ctx, fmt.Sprintf(` + SELECT CASE WHEN m > 0 THEN setval(pg_get_serial_sequence($1, $2), m, true) + ELSE setval(pg_get_serial_sequence($1, $2), 1, false) END + FROM (SELECT COALESCE(max(%s), 0) AS m FROM %s) AS top`, col, ident), table, c.name); err != nil { return fmt.Errorf("backup: carry on %s's sequence: %w", table, err) } diff --git a/internal/db/dbq/library_changes.sql.go b/internal/db/dbq/library_changes.sql.go index 1d38222d..bed9d3a2 100644 --- a/internal/db/dbq/library_changes.sql.go +++ b/internal/db/dbq/library_changes.sql.go @@ -26,6 +26,20 @@ func (q *Queries) DeleteOldLibraryChanges(ctx context.Context, changedAt pgtype. return result.RowsAffected(), nil } +const getLastIssuedLibraryChangeCursor = `-- name: GetLastIssuedLibraryChangeCursor :one +SELECT COALESCE(pg_sequence_last_value(pg_get_serial_sequence('library_changes', 'id')::regclass), 0)::BIGINT +` + +// The highest cursor this server has handed out, even once the compactor has +// trimmed the rows: the id sequence's last value. A restored server's +// sequence stands at the backup's last change. +func (q *Queries) GetLastIssuedLibraryChangeCursor(ctx context.Context) (int64, error) { + row := q.db.QueryRow(ctx, getLastIssuedLibraryChangeCursor) + var column_1 int64 + err := row.Scan(&column_1) + return column_1, err +} + const getLibraryChangesSince = `-- name: GetLibraryChangesSince :many SELECT id, entity_type, entity_id, op, changed_at FROM library_changes diff --git a/internal/db/queries/library_changes.sql b/internal/db/queries/library_changes.sql index 0437b36a..3b9f7733 100644 --- a/internal/db/queries/library_changes.sql +++ b/internal/db/queries/library_changes.sql @@ -22,6 +22,12 @@ LIMIT $2; -- name: GetMaxLibraryChangeCursor :one SELECT COALESCE(MAX(id), 0)::BIGINT FROM library_changes; +-- name: GetLastIssuedLibraryChangeCursor :one +-- The highest cursor this server has handed out, even once the compactor has +-- trimmed the rows: the id sequence's last value. A restored server's +-- sequence stands at the backup's last change. +SELECT COALESCE(pg_sequence_last_value(pg_get_serial_sequence('library_changes', 'id')::regclass), 0)::BIGINT; + -- name: GetMinLibraryChangeCursor :one SELECT COALESCE(MIN(id), 0)::BIGINT FROM library_changes; diff --git a/internal/server/hygiene.go b/internal/server/hygiene.go index 854e85d3..a820482d 100644 --- a/internal/server/hygiene.go +++ b/internal/server/hygiene.go @@ -18,6 +18,23 @@ const maxRequestBody = 4 << 20 // bodyReadTimeout bounds how long a client may take to send a request body. const bodyReadTimeout = 30 * time.Second +// restorePath takes a whole backup file, so it gets room for one: the file is +// the database alone (the music travels separately), which is tens of MB for +// a large library. +const ( + restorePath = "/api/auth/restore" + maxRestoreBody = 1 << 30 + restoreReadTimeout = 15 * time.Minute +) + +// bodyLimits is the size and time a request may take to send its body. +func bodyLimits(r *http.Request) (int64, time.Duration) { + if r.Method == http.MethodPost && r.URL.Path == restorePath { + return maxRestoreBody, restoreReadTimeout + } + return maxRequestBody, bodyReadTimeout +} + // limitRequestBody caps the body size and the time allowed to deliver it. // // Why not http.Server.ReadTimeout: that deadline stays armed for the whole @@ -35,8 +52,9 @@ func limitRequestBody(next http.Handler) http.Handler { rc := http.NewResponseController(w) // Best-effort: a ResponseWriter that can't set deadlines (a test // recorder) still gets the size limit. - armed := rc.SetReadDeadline(time.Now().Add(bodyReadTimeout)) == nil - body := http.MaxBytesReader(w, r.Body, maxRequestBody) + size, wait := bodyLimits(r) + armed := rc.SetReadDeadline(time.Now().Add(wait)) == nil + body := http.MaxBytesReader(w, r.Body, size) if armed { body = &deadlineClearingBody{ReadCloser: body, rc: rc} } @@ -83,6 +101,10 @@ func (b *deadlineClearingBody) Close() error { // attaches on its own, so they are not checked. The Android app does send // the cookie, but every body it sends is JSON (Retrofit's kotlinx converter) // and its bodiless writes carry no Content-Type, so it passes unchanged. +// +// The restore upload is a backup file, not JSON. It is safe without the check: +// it acts only with the setup token in a request header, which a form cannot +// send and a cross-site fetch cannot send without a preflight. func requireJSONForCookieWrites(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.Method { @@ -90,7 +112,7 @@ func requireJSONForCookieWrites(next http.Handler) http.Handler { next.ServeHTTP(w, r) return } - if !strings.HasPrefix(r.URL.Path, "/api/") { + if !strings.HasPrefix(r.URL.Path, "/api/") || r.URL.Path == restorePath { next.ServeHTTP(w, r) return } diff --git a/internal/server/hygiene_test.go b/internal/server/hygiene_test.go index 2c31dbf6..2dec4769 100644 --- a/internal/server/hygiene_test.go +++ b/internal/server/hygiene_test.go @@ -40,6 +40,21 @@ func TestLimitRequestBody_RejectsOversizedBody(t *testing.T) { } } +// A backup file is far larger than any other body, and only the restore +// upload may send one. +func TestLimitRequestBody_TheRestoreUploadTakesABackupFile(t *testing.T) { + h := limitRequestBody(okHandler()) + file := strings.NewReader(strings.Repeat("x", maxRequestBody*2)) + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodPost, restorePath, file)) + if w.Code != http.StatusNoContent { + t.Errorf("a backup-sized restore upload: status = %d, want 204", w.Code) + } + if size, wait := bodyLimits(httptest.NewRequest(http.MethodPost, "/api/playlists", nil)); size != maxRequestBody || wait != bodyReadTimeout { + t.Errorf("another path's limits = %d, %v; want the ordinary ones", size, wait) + } +} + // The cases a cross-site form or no-preflight fetch could produce must be // refused when the session cookie rides along; everything a real client // sends must pass. @@ -55,6 +70,7 @@ func TestRequireJSONForCookieWrites(t *testing.T) { want int }{ {"form post with cookie", http.MethodPost, "/api/me/password", "application/x-www-form-urlencoded", "a=b", true, http.StatusUnsupportedMediaType}, + {"restore upload with a stale cookie", http.MethodPost, "/api/auth/restore", "application/x-tar", "tar", true, http.StatusNoContent}, {"text/plain post with cookie", http.MethodPost, "/api/me/password", "text/plain", `{"a":1}`, true, http.StatusUnsupportedMediaType}, {"multipart with cookie", http.MethodPut, "/api/me/profile", "multipart/form-data; boundary=x", "--x--", true, http.StatusUnsupportedMediaType}, {"body without content type", http.MethodPost, "/api/me/profile", "", `{"a":1}`, true, http.StatusUnsupportedMediaType}, diff --git a/web/src/lib/auth/store.svelte.ts b/web/src/lib/auth/store.svelte.ts index 777f4129..efb626a8 100644 --- a/web/src/lib/auth/store.svelte.ts +++ b/web/src/lib/auth/store.svelte.ts @@ -1,4 +1,4 @@ -import { api, type User, type LoginResponse } from '#lib/api/client.js'; +import { api, apiFetch, type User, type LoginResponse } from '#lib/api/client.js'; import { putMyTimezone } from '#lib/api/me.js'; import { queryClient } from '#lib/query/client.js'; import { signalSessionEnd } from './sessionEnd.svelte'; @@ -75,6 +75,20 @@ export async function register(opts: { void sendTimezoneIfStale(); } +export type RestoreResult = { accounts: number; scan_started: boolean }; + +/** + * Loads a backup file into a new server, which has no accounts yet. The file + * is the request body; the setup token from the server log goes in a header. + */ +export async function restoreBackup(file: Blob, setupToken: string): Promise { + return (await apiFetch('/api/auth/restore', { + method: 'POST', + body: file, + headers: { 'Content-Type': 'application/x-tar', 'X-Setup-Token': setupToken } + })) as RestoreResult; +} + export async function forgotPassword(email: string): Promise { await api.post('/api/auth/forgot-password', { email }); } diff --git a/web/src/lib/components/auth/RestoreForm.svelte b/web/src/lib/components/auth/RestoreForm.svelte new file mode 100644 index 00000000..f77c47a4 --- /dev/null +++ b/web/src/lib/components/auth/RestoreForm.svelte @@ -0,0 +1,94 @@ + + +{#if done} +
+

+ Restored {plural(done.accounts, 'account', 'accounts')}. Sign in with the password you had. +

+

+ {done.scan_started + ? 'The library scan has started: it finds each music file and fetches the artwork and tags again.' + : 'Start a library scan under Admin › Library once you’re signed in.'} +

+ Sign in +
+{:else} +
+ + + + {#if error} + + {/if} +
+{/if} diff --git a/web/src/routes/register/+page.svelte b/web/src/routes/register/+page.svelte index 86259e5e..1f9f4be1 100644 --- a/web/src/routes/register/+page.svelte +++ b/web/src/routes/register/+page.svelte @@ -5,6 +5,7 @@ import { getSetupStatus, register } from '#lib/auth/store.svelte.js'; import { errCode } from '#lib/api/errors.js'; import { rateLimitMessage } from '#lib/api/client.js'; + import RestoreForm from '#lib/components/auth/RestoreForm.svelte'; let username = $state(''); let password = $state(''); @@ -14,6 +15,8 @@ // True on a brand-new server: the first account becomes admin and must // carry the setup token from the server log. let setupRequired = $state(false); + // A new server can instead start from a backup of another Minstrel. + let restoring = $state(false); onMount(async () => { try { @@ -76,7 +79,24 @@
-

Create account

+

{restoring ? 'Restore a backup' : 'Create account'}

+ {#if setupRequired} +

+ {#if restoring} + + {:else} + Moving from another Minstrel? + + {/if} +

+ {/if} + {#if restoring} + + {:else}