feat: Minstrel backs itself up nightly to a folder or an S3 bucket, off until an admin switches it on (#6773)

Admin › Backups shows a verdict first (off, not set up, waiting, backed up,
overdue, failing since when), Back up now, Test destination and the latest
runs, with the settings beside them.

- backup.Service: one run at a time, bounded at 30 minutes; the schedule
  ticks every 5 minutes, catches up a slot missed while the server was down,
  and retries a failure after an hour and then after as long as it has been
  failing. Runs are recorded in backup_runs, pruned after 90 days; a run cut
  off by a shutdown is marked failed at the next start.
- A folder destination writes database/minstrel-<UTC stamp>.tar and refuses
  a folder that isn't there. An S3 destination stages the file, hashes it,
  and uploads it under database/ through a throttled reader (4 MiB/s by
  default, 0 for no limit).
- Test destination checks the form as typed and only lists.
- The bucket secret is sealed like every stored credential; backup_settings
  and backup_runs are left out of the backup itself.
- The panel polls only while a run is going, capped at the run limit plus
  five minutes.

The music switch named in the task is left to #6774, so no control ships
that does nothing.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/db/dbq/backups.sql.go

@@ -0,0 +1,217 @@
+// Code generated by sqlc. DO NOT EDIT.
+// versions:
+//   sqlc v1.31.1
+// source: backups.sql
+
+package dbq
+
+import (
+	"context"
+
+	"github.com/jackc/pgx/v5/pgtype"
+)
+
+const backupRunTimes = `-- name: BackupRunTimes :one
+SELECT
+    COALESCE((SELECT max(started_at) FROM backup_runs WHERE status = 'succeeded'),
+             'epoch'::timestamptz)::timestamptz AS last_success,
+    COALESCE((SELECT max(started_at) FROM backup_runs),
+             'epoch'::timestamptz)::timestamptz AS last_attempt,
+    COALESCE((SELECT min(started_at) FROM backup_runs
+               WHERE status = 'failed'
+                 AND started_at > COALESCE((SELECT max(started_at) FROM backup_runs WHERE status = 'succeeded'),
+                                           'epoch'::timestamptz)),
+             'epoch'::timestamptz)::timestamptz AS failing_since
+`
+
+type BackupRunTimesRow struct {
+	LastSuccess  pgtype.Timestamptz
+	LastAttempt  pgtype.Timestamptz
+	FailingSince pgtype.Timestamptz
+}
+
+// The times the verdict turns on. A run that never happened is the epoch,
+// so the row always exists. failing_since is the first failure after the last
+// success, and the epoch when the latest finished run succeeded.
+func (q *Queries) BackupRunTimes(ctx context.Context) (BackupRunTimesRow, error) {
+	row := q.db.QueryRow(ctx, backupRunTimes)
+	var i BackupRunTimesRow
+	err := row.Scan(&i.LastSuccess, &i.LastAttempt, &i.FailingSince)
+	return i, err
+}
+
+const failUnfinishedBackupRuns = `-- name: FailUnfinishedBackupRuns :execrows
+UPDATE backup_runs
+   SET status = 'failed', finished_at = now(),
+       error = 'The server stopped while the backup was running.'
+ WHERE status = 'running'
+`
+
+func (q *Queries) FailUnfinishedBackupRuns(ctx context.Context) (int64, error) {
+	result, err := q.db.Exec(ctx, failUnfinishedBackupRuns)
+	if err != nil {
+		return 0, err
+	}
+	return result.RowsAffected(), nil
+}
+
+const finishBackupRun = `-- name: FinishBackupRun :exec
+UPDATE backup_runs
+   SET status      = $2,
+       finished_at = now(),
+       location    = $3,
+       bytes       = $4,
+       error       = $5
+ WHERE id = $1
+`
+
+type FinishBackupRunParams struct {
+	ID       pgtype.UUID
+	Status   string
+	Location string
+	Bytes    int64
+	Error    string
+}
+
+func (q *Queries) FinishBackupRun(ctx context.Context, arg FinishBackupRunParams) error {
+	_, err := q.db.Exec(ctx, finishBackupRun,
+		arg.ID,
+		arg.Status,
+		arg.Location,
+		arg.Bytes,
+		arg.Error,
+	)
+	return err
+}
+
+const getBackupSettings = `-- name: GetBackupSettings :one
+
+SELECT id, enabled, destination, folder, endpoint, bucket, access_key, secret_key, run_hour, upload_limit_kib, updated_at FROM backup_settings WHERE id = true
+`
+
+// Minstrel's own backup: its settings and its runs.
+func (q *Queries) GetBackupSettings(ctx context.Context) (BackupSetting, error) {
+	row := q.db.QueryRow(ctx, getBackupSettings)
+	var i BackupSetting
+	err := row.Scan(
+		&i.ID,
+		&i.Enabled,
+		&i.Destination,
+		&i.Folder,
+		&i.Endpoint,
+		&i.Bucket,
+		&i.AccessKey,
+		&i.SecretKey,
+		&i.RunHour,
+		&i.UploadLimitKib,
+		&i.UpdatedAt,
+	)
+	return i, err
+}
+
+const listBackupRuns = `-- name: ListBackupRuns :many
+SELECT id, trigger, status, started_at, finished_at, location, bytes, error
+  FROM backup_runs
+ ORDER BY started_at DESC
+ LIMIT $1
+`
+
+func (q *Queries) ListBackupRuns(ctx context.Context, limit int32) ([]BackupRun, error) {
+	rows, err := q.db.Query(ctx, listBackupRuns, limit)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var items []BackupRun
+	for rows.Next() {
+		var i BackupRun
+		if err := rows.Scan(
+			&i.ID,
+			&i.Trigger,
+			&i.Status,
+			&i.StartedAt,
+			&i.FinishedAt,
+			&i.Location,
+			&i.Bytes,
+			&i.Error,
+		); err != nil {
+			return nil, err
+		}
+		items = append(items, i)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return items, nil
+}
+
+const pruneBackupRuns = `-- name: PruneBackupRuns :execrows
+DELETE FROM backup_runs WHERE status <> 'running' AND started_at < $1
+`
+
+func (q *Queries) PruneBackupRuns(ctx context.Context, startedAt pgtype.Timestamptz) (int64, error) {
+	result, err := q.db.Exec(ctx, pruneBackupRuns, startedAt)
+	if err != nil {
+		return 0, err
+	}
+	return result.RowsAffected(), nil
+}
+
+const saveBackupSettings = `-- name: SaveBackupSettings :exec
+UPDATE backup_settings
+   SET enabled     = $1,
+       destination = $2,
+       folder      = $3,
+       endpoint    = $4,
+       bucket      = $5,
+       access_key  = $6,
+       secret_key  = $7,
+       run_hour    = $8,
+       upload_limit_kib = $9,
+       updated_at  = now()
+ WHERE id = true
+`
+
+type SaveBackupSettingsParams struct {
+	Enabled        bool
+	Destination    string
+	Folder         string
+	Endpoint       string
+	Bucket         string
+	AccessKey      string
+	SecretKey      string
+	RunHour        int16
+	UploadLimitKib int32
+}
+
+func (q *Queries) SaveBackupSettings(ctx context.Context, arg SaveBackupSettingsParams) error {
+	_, err := q.db.Exec(ctx, saveBackupSettings,
+		arg.Enabled,
+		arg.Destination,
+		arg.Folder,
+		arg.Endpoint,
+		arg.Bucket,
+		arg.AccessKey,
+		arg.SecretKey,
+		arg.RunHour,
+		arg.UploadLimitKib,
+	)
+	return err
+}
+
+const startBackupRun = `-- name: StartBackupRun :one
+INSERT INTO backup_runs (trigger) VALUES ($1)
+RETURNING id, started_at
+`
+
+type StartBackupRunRow struct {
+	ID        pgtype.UUID
+	StartedAt pgtype.Timestamptz
+}
+
+func (q *Queries) StartBackupRun(ctx context.Context, trigger string) (StartBackupRunRow, error) {
+	row := q.db.QueryRow(ctx, startBackupRun, trigger)
+	var i StartBackupRunRow
+	err := row.Scan(&i.ID, &i.StartedAt)
+	return i, err
+}

internal/db/dbq/models.go

@@ -305,6 +305,31 @@ type AuditLog struct {
 	CreatedAt pgtype.Timestamptz
 }
 
+type BackupRun struct {
+	ID         pgtype.UUID
+	Trigger    string
+	Status     string
+	StartedAt  pgtype.Timestamptz
+	FinishedAt pgtype.Timestamptz
+	Location   string
+	Bytes      int64
+	Error      string
+}
+
+type BackupSetting struct {
+	ID             bool
+	Enabled        bool
+	Destination    string
+	Folder         string
+	Endpoint       string
+	Bucket         string
+	AccessKey      string
+	SecretKey      string
+	RunHour        int16
+	UploadLimitKib int32
+	UpdatedAt      pgtype.Timestamptz
+}
+
 type CandidateArtistTag struct {
 	CandidateMbid string
 	Tag           string