feat: Minstrel backs itself up nightly to a folder or an S3 bucket, off until an admin switches it on (#6773)

Admin › Backups shows a verdict first (off, not set up, waiting, backed up,
overdue, failing since when), Back up now, Test destination and the latest
runs, with the settings beside them.

- backup.Service: one run at a time, bounded at 30 minutes; the schedule
  ticks every 5 minutes, catches up a slot missed while the server was down,
  and retries a failure after an hour and then after as long as it has been
  failing. Runs are recorded in backup_runs, pruned after 90 days; a run cut
  off by a shutdown is marked failed at the next start.
- A folder destination writes database/minstrel-<UTC stamp>.tar and refuses
  a folder that isn't there. An S3 destination stages the file, hashes it,
  and uploads it under database/ through a throttled reader (4 MiB/s by
  default, 0 for no limit).
- Test destination checks the form as typed and only lists.
- The bucket secret is sealed like every stored credential; backup_settings
  and backup_runs are left out of the backup itself.
- The panel polls only while a run is going, capped at the run limit plus
  five minutes.

The music switch named in the task is left to #6774, so no control ships
that does nothing.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/db/dbq/backups.sql.go

@@ -0,0 +1,217 @@
+// Code generated by sqlc. DO NOT EDIT.
+// versions:
+//   sqlc v1.31.1
+// source: backups.sql
+
+package dbq
+
+import (
+	"context"
+
+	"github.com/jackc/pgx/v5/pgtype"
+)
+
+const backupRunTimes = `-- name: BackupRunTimes :one
+SELECT
+    COALESCE((SELECT max(started_at) FROM backup_runs WHERE status = 'succeeded'),
+             'epoch'::timestamptz)::timestamptz AS last_success,
+    COALESCE((SELECT max(started_at) FROM backup_runs),
+             'epoch'::timestamptz)::timestamptz AS last_attempt,
+    COALESCE((SELECT min(started_at) FROM backup_runs
+               WHERE status = 'failed'
+                 AND started_at > COALESCE((SELECT max(started_at) FROM backup_runs WHERE status = 'succeeded'),
+                                           'epoch'::timestamptz)),
+             'epoch'::timestamptz)::timestamptz AS failing_since
+`
+
+type BackupRunTimesRow struct {
+	LastSuccess  pgtype.Timestamptz
+	LastAttempt  pgtype.Timestamptz
+	FailingSince pgtype.Timestamptz
+}
+
+// The times the verdict turns on. A run that never happened is the epoch,
+// so the row always exists. failing_since is the first failure after the last
+// success, and the epoch when the latest finished run succeeded.
+func (q *Queries) BackupRunTimes(ctx context.Context) (BackupRunTimesRow, error) {
+	row := q.db.QueryRow(ctx, backupRunTimes)
+	var i BackupRunTimesRow
+	err := row.Scan(&i.LastSuccess, &i.LastAttempt, &i.FailingSince)
+	return i, err
+}
+
+const failUnfinishedBackupRuns = `-- name: FailUnfinishedBackupRuns :execrows
+UPDATE backup_runs
+   SET status = 'failed', finished_at = now(),
+       error = 'The server stopped while the backup was running.'
+ WHERE status = 'running'
+`
+
+func (q *Queries) FailUnfinishedBackupRuns(ctx context.Context) (int64, error) {
+	result, err := q.db.Exec(ctx, failUnfinishedBackupRuns)
+	if err != nil {
+		return 0, err
+	}
+	return result.RowsAffected(), nil
+}
+
+const finishBackupRun = `-- name: FinishBackupRun :exec
+UPDATE backup_runs
+   SET status      = $2,
+       finished_at = now(),
+       location    = $3,
+       bytes       = $4,
+       error       = $5
+ WHERE id = $1
+`
+
+type FinishBackupRunParams struct {
+	ID       pgtype.UUID
+	Status   string
+	Location string
+	Bytes    int64
+	Error    string
+}
+
+func (q *Queries) FinishBackupRun(ctx context.Context, arg FinishBackupRunParams) error {
+	_, err := q.db.Exec(ctx, finishBackupRun,
+		arg.ID,
+		arg.Status,
+		arg.Location,
+		arg.Bytes,
+		arg.Error,
+	)
+	return err
+}
+
+const getBackupSettings = `-- name: GetBackupSettings :one
+
+SELECT id, enabled, destination, folder, endpoint, bucket, access_key, secret_key, run_hour, upload_limit_kib, updated_at FROM backup_settings WHERE id = true
+`
+
+// Minstrel's own backup: its settings and its runs.
+func (q *Queries) GetBackupSettings(ctx context.Context) (BackupSetting, error) {
+	row := q.db.QueryRow(ctx, getBackupSettings)
+	var i BackupSetting
+	err := row.Scan(
+		&i.ID,
+		&i.Enabled,
+		&i.Destination,
+		&i.Folder,
+		&i.Endpoint,
+		&i.Bucket,
+		&i.AccessKey,
+		&i.SecretKey,
+		&i.RunHour,
+		&i.UploadLimitKib,
+		&i.UpdatedAt,
+	)
+	return i, err
+}
+
+const listBackupRuns = `-- name: ListBackupRuns :many
+SELECT id, trigger, status, started_at, finished_at, location, bytes, error
+  FROM backup_runs
+ ORDER BY started_at DESC
+ LIMIT $1
+`
+
+func (q *Queries) ListBackupRuns(ctx context.Context, limit int32) ([]BackupRun, error) {
+	rows, err := q.db.Query(ctx, listBackupRuns, limit)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var items []BackupRun
+	for rows.Next() {
+		var i BackupRun
+		if err := rows.Scan(
+			&i.ID,
+			&i.Trigger,
+			&i.Status,
+			&i.StartedAt,
+			&i.FinishedAt,
+			&i.Location,
+			&i.Bytes,
+			&i.Error,
+		); err != nil {
+			return nil, err
+		}
+		items = append(items, i)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return items, nil
+}
+
+const pruneBackupRuns = `-- name: PruneBackupRuns :execrows
+DELETE FROM backup_runs WHERE status <> 'running' AND started_at < $1
+`
+
+func (q *Queries) PruneBackupRuns(ctx context.Context, startedAt pgtype.Timestamptz) (int64, error) {
+	result, err := q.db.Exec(ctx, pruneBackupRuns, startedAt)
+	if err != nil {
+		return 0, err
+	}
+	return result.RowsAffected(), nil
+}
+
+const saveBackupSettings = `-- name: SaveBackupSettings :exec
+UPDATE backup_settings
+   SET enabled     = $1,
+       destination = $2,
+       folder      = $3,
+       endpoint    = $4,
+       bucket      = $5,
+       access_key  = $6,
+       secret_key  = $7,
+       run_hour    = $8,
+       upload_limit_kib = $9,
+       updated_at  = now()
+ WHERE id = true
+`
+
+type SaveBackupSettingsParams struct {
+	Enabled        bool
+	Destination    string
+	Folder         string
+	Endpoint       string
+	Bucket         string
+	AccessKey      string
+	SecretKey      string
+	RunHour        int16
+	UploadLimitKib int32
+}
+
+func (q *Queries) SaveBackupSettings(ctx context.Context, arg SaveBackupSettingsParams) error {
+	_, err := q.db.Exec(ctx, saveBackupSettings,
+		arg.Enabled,
+		arg.Destination,
+		arg.Folder,
+		arg.Endpoint,
+		arg.Bucket,
+		arg.AccessKey,
+		arg.SecretKey,
+		arg.RunHour,
+		arg.UploadLimitKib,
+	)
+	return err
+}
+
+const startBackupRun = `-- name: StartBackupRun :one
+INSERT INTO backup_runs (trigger) VALUES ($1)
+RETURNING id, started_at
+`
+
+type StartBackupRunRow struct {
+	ID        pgtype.UUID
+	StartedAt pgtype.Timestamptz
+}
+
+func (q *Queries) StartBackupRun(ctx context.Context, trigger string) (StartBackupRunRow, error) {
+	row := q.db.QueryRow(ctx, startBackupRun, trigger)
+	var i StartBackupRunRow
+	err := row.Scan(&i.ID, &i.StartedAt)
+	return i, err
+}

internal/db/dbq/models.go

@@ -305,6 +305,31 @@ type AuditLog struct {
 	CreatedAt pgtype.Timestamptz
 }
 
+type BackupRun struct {
+	ID         pgtype.UUID
+	Trigger    string
+	Status     string
+	StartedAt  pgtype.Timestamptz
+	FinishedAt pgtype.Timestamptz
+	Location   string
+	Bytes      int64
+	Error      string
+}
+
+type BackupSetting struct {
+	ID             bool
+	Enabled        bool
+	Destination    string
+	Folder         string
+	Endpoint       string
+	Bucket         string
+	AccessKey      string
+	SecretKey      string
+	RunHour        int16
+	UploadLimitKib int32
+	UpdatedAt      pgtype.Timestamptz
+}
+
 type CandidateArtistTag struct {
 	CandidateMbid string
 	Tag           string

internal/db/migrations/0099_backups.down.sql

@@ -0,0 +1,2 @@
+DROP TABLE IF EXISTS backup_runs;
+DROP TABLE IF EXISTS backup_settings;

internal/db/migrations/0099_backups.up.sql

@@ -0,0 +1,44 @@
+-- 0099_backups.up.sql — Minstrel backing itself up.
+--
+--   backup_settings  one row: whether the nightly backup is on, where it goes
+--                    (a folder, or an S3 bucket such as a Fabled Archives
+--                    target), and the hour it runs, in UTC. Off until an
+--                    admin switches it on.
+--   backup_runs      one row per attempt: when, what started it, how it ended,
+--                    where the file went, its size, and the error in words.
+CREATE TABLE backup_settings (
+    id          boolean PRIMARY KEY DEFAULT true CONSTRAINT backup_settings_one_row CHECK (id),
+    enabled     boolean NOT NULL DEFAULT false,
+    destination text NOT NULL DEFAULT 's3'
+        CONSTRAINT backup_settings_destination_check CHECK (destination IN ('folder', 's3')),
+    folder      text NOT NULL DEFAULT '',
+    endpoint    text NOT NULL DEFAULT '',
+    bucket      text NOT NULL DEFAULT '',
+    access_key  text NOT NULL DEFAULT '',
+    -- Sealed under the stream secret, as every stored credential is.
+    secret_key  text NOT NULL DEFAULT '',
+    run_hour    smallint NOT NULL DEFAULT 3
+        CONSTRAINT backup_settings_run_hour_check CHECK (run_hour BETWEEN 0 AND 23),
+    -- The most an upload to the bucket may send, in KiB a second; 0 is no limit.
+    upload_limit_kib integer NOT NULL DEFAULT 4096
+        CONSTRAINT backup_settings_upload_limit_check CHECK (upload_limit_kib >= 0),
+    updated_at  timestamptz NOT NULL DEFAULT now()
+);
+
+INSERT INTO backup_settings DEFAULT VALUES;
+
+CREATE TABLE backup_runs (
+    id          uuid PRIMARY KEY DEFAULT gen_random_uuid(),
+    trigger     text NOT NULL
+        CONSTRAINT backup_runs_trigger_check CHECK (trigger IN ('schedule', 'manual')),
+    status      text NOT NULL DEFAULT 'running'
+        CONSTRAINT backup_runs_status_check CHECK (status IN ('running', 'succeeded', 'failed')),
+    started_at  timestamptz NOT NULL DEFAULT now(),
+    finished_at timestamptz,
+    -- Where the file went: the object key, or the file's path in the folder.
+    location    text NOT NULL DEFAULT '',
+    bytes       bigint NOT NULL DEFAULT 0,
+    error       text NOT NULL DEFAULT ''
+);
+
+CREATE INDEX backup_runs_started_at_idx ON backup_runs (started_at DESC);

internal/db/queries/backups.sql

@@ -0,0 +1,61 @@
+-- Minstrel's own backup: its settings and its runs.
+
+-- name: GetBackupSettings :one
+SELECT * FROM backup_settings WHERE id = true;
+
+-- name: SaveBackupSettings :exec
+UPDATE backup_settings
+   SET enabled     = $1,
+       destination = $2,
+       folder      = $3,
+       endpoint    = $4,
+       bucket      = $5,
+       access_key  = $6,
+       secret_key  = $7,
+       run_hour    = $8,
+       upload_limit_kib = $9,
+       updated_at  = now()
+ WHERE id = true;
+
+-- name: StartBackupRun :one
+INSERT INTO backup_runs (trigger) VALUES ($1)
+RETURNING id, started_at;
+
+-- name: FinishBackupRun :exec
+UPDATE backup_runs
+   SET status      = $2,
+       finished_at = now(),
+       location    = $3,
+       bytes       = $4,
+       error       = $5
+ WHERE id = $1;
+
+-- name: ListBackupRuns :many
+SELECT id, trigger, status, started_at, finished_at, location, bytes, error
+  FROM backup_runs
+ ORDER BY started_at DESC
+ LIMIT $1;
+
+-- name: BackupRunTimes :one
+-- The times the verdict turns on. A run that never happened is the epoch,
+-- so the row always exists. failing_since is the first failure after the last
+-- success, and the epoch when the latest finished run succeeded.
+SELECT
+    COALESCE((SELECT max(started_at) FROM backup_runs WHERE status = 'succeeded'),
+             'epoch'::timestamptz)::timestamptz AS last_success,
+    COALESCE((SELECT max(started_at) FROM backup_runs),
+             'epoch'::timestamptz)::timestamptz AS last_attempt,
+    COALESCE((SELECT min(started_at) FROM backup_runs
+               WHERE status = 'failed'
+                 AND started_at > COALESCE((SELECT max(started_at) FROM backup_runs WHERE status = 'succeeded'),
+                                           'epoch'::timestamptz)),
+             'epoch'::timestamptz)::timestamptz AS failing_since;
+
+-- name: FailUnfinishedBackupRuns :execrows
+UPDATE backup_runs
+   SET status = 'failed', finished_at = now(),
+       error = 'The server stopped while the backup was running.'
+ WHERE status = 'running';
+
+-- name: PruneBackupRuns :execrows
+DELETE FROM backup_runs WHERE status <> 'running' AND started_at < $1;