feat: the backup can include the music, sending each library file once to music/ and again only when it changes (#6774)

With "Include the music" on, each night's run sends the library files the
destination doesn't hold yet, after the database file has landed, for up to
six hours; a large library spans several nights the first time. Files keep
their path under the library folders' common root, beside database/ in the
same folder or bucket.

backup_music_files records each file's size and mtime as sent, so a night
sends only new and changed files, and a rescan that touches a track but finds
the same file sends nothing. A changed file goes to its old name; Minstrel
never deletes there. An object the bucket refuses (a locked overwrite) is
kept with its answer and goes again once the file changes or the settings are
saved; three refusals in a row stop the night's music, since that is a key
that may not write under music/. A folder copy is written under a .part name
and renamed, keeping the file's time.

The panel shows how much of the library is in, the refused files, and each
run's music count and note. The verdict still turns on the database file.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/db/dbq/backups.sql.go

@@ -11,6 +11,39 @@ import (
 	"github.com/jackc/pgx/v5/pgtype"
 )
 
+const backupMusicProgress = `-- name: BackupMusicProgress :one
+SELECT count(*)::bigint AS total_files,
+       COALESCE(sum(t.file_size), 0)::bigint AS total_bytes,
+       count(*) FILTER (WHERE m.error = '' AND m.size = t.file_size AND t.updated_at <= m.checked_at)::bigint AS sent_files,
+       COALESCE(sum(t.file_size) FILTER (WHERE m.error = '' AND m.size = t.file_size AND t.updated_at <= m.checked_at), 0)::bigint AS sent_bytes,
+       count(*) FILTER (WHERE m.error <> '' AND m.size = t.file_size AND t.updated_at <= m.checked_at)::bigint AS refused_files
+  FROM tracks t
+  LEFT JOIN backup_music_files m ON m.track_id = t.id
+ WHERE t.missing_since IS NULL
+`
+
+type BackupMusicProgressRow struct {
+	TotalFiles   int64
+	TotalBytes   int64
+	SentFiles    int64
+	SentBytes    int64
+	RefusedFiles int64
+}
+
+// How much of the library the backup holds as it is now.
+func (q *Queries) BackupMusicProgress(ctx context.Context) (BackupMusicProgressRow, error) {
+	row := q.db.QueryRow(ctx, backupMusicProgress)
+	var i BackupMusicProgressRow
+	err := row.Scan(
+		&i.TotalFiles,
+		&i.TotalBytes,
+		&i.SentFiles,
+		&i.SentBytes,
+		&i.RefusedFiles,
+	)
+	return i, err
+}
+
 const backupRunTimes = `-- name: BackupRunTimes :one
 SELECT
     COALESCE((SELECT max(started_at) FROM backup_runs WHERE status = 'succeeded'),
@@ -61,16 +94,22 @@ UPDATE backup_runs
        finished_at = now(),
        location    = $3,
        bytes       = $4,
-       error       = $5
+       error       = $5,
+       music_files = $6,
+       music_bytes = $7,
+       music_note  = $8
  WHERE id = $1
 `
 
 type FinishBackupRunParams struct {
-	ID       pgtype.UUID
-	Status   string
-	Location string
-	Bytes    int64
-	Error    string
+	ID         pgtype.UUID
+	Status     string
+	Location   string
+	Bytes      int64
+	Error      string
+	MusicFiles int32
+	MusicBytes int64
+	MusicNote  string
 }
 
 func (q *Queries) FinishBackupRun(ctx context.Context, arg FinishBackupRunParams) error {
@@ -80,13 +119,16 @@ func (q *Queries) FinishBackupRun(ctx context.Context, arg FinishBackupRunParams
 		arg.Location,
 		arg.Bytes,
 		arg.Error,
+		arg.MusicFiles,
+		arg.MusicBytes,
+		arg.MusicNote,
 	)
 	return err
 }
 
 const getBackupSettings = `-- name: GetBackupSettings :one
 
-SELECT id, enabled, destination, folder, endpoint, bucket, access_key, secret_key, run_hour, upload_limit_kib, updated_at FROM backup_settings WHERE id = true
+SELECT id, enabled, destination, folder, endpoint, bucket, access_key, secret_key, run_hour, upload_limit_kib, updated_at, include_music FROM backup_settings WHERE id = true
 `
 
 // Minstrel's own backup: its settings and its runs.
@@ -105,12 +147,14 @@ func (q *Queries) GetBackupSettings(ctx context.Context) (BackupSetting, error)
 		&i.RunHour,
 		&i.UploadLimitKib,
 		&i.UpdatedAt,
+		&i.IncludeMusic,
 	)
 	return i, err
 }
 
 const listBackupRuns = `-- name: ListBackupRuns :many
-SELECT id, trigger, status, started_at, finished_at, location, bytes, error
+SELECT id, trigger, status, started_at, finished_at, location, bytes, error,
+       music_files, music_bytes, music_note
   FROM backup_runs
  ORDER BY started_at DESC
  LIMIT $1
@@ -134,6 +178,69 @@ func (q *Queries) ListBackupRuns(ctx context.Context, limit int32) ([]BackupRun,
 			&i.Location,
 			&i.Bytes,
 			&i.Error,
+			&i.MusicFiles,
+			&i.MusicBytes,
+			&i.MusicNote,
+		); err != nil {
+			return nil, err
+		}
+		items = append(items, i)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return items, nil
+}
+
+const pendingBackupMusic = `-- name: PendingBackupMusic :many
+SELECT t.id, t.file_path, t.file_size,
+       m.size AS sent_size, m.mtime AS sent_mtime, m.error AS sent_error
+  FROM tracks t
+  LEFT JOIN backup_music_files m ON m.track_id = t.id
+ WHERE t.missing_since IS NULL
+   AND t.file_path > $1::text
+   AND (m.track_id IS NULL OR m.size <> t.file_size OR t.updated_at > m.checked_at
+        OR (m.error <> '' AND m.checked_at < (SELECT updated_at FROM backup_settings)))
+ ORDER BY t.file_path
+ LIMIT $2
+`
+
+type PendingBackupMusicParams struct {
+	AfterPath string
+	MaxRows   int32
+}
+
+type PendingBackupMusicRow struct {
+	ID        pgtype.UUID
+	FilePath  string
+	FileSize  int64
+	SentSize  *int64
+	SentMtime pgtype.Timestamptz
+	SentError *string
+}
+
+// Library files the backup may not hold as they are now, in path order from
+// after_path, so a night that runs out of time carries on where it stopped:
+// never sent, a different size, touched by a scan since they were last
+// checked, or refused before the settings were last saved (a fixed key gets
+// another go). The caller compares the file's own size and mtime before
+// sending.
+func (q *Queries) PendingBackupMusic(ctx context.Context, arg PendingBackupMusicParams) ([]PendingBackupMusicRow, error) {
+	rows, err := q.db.Query(ctx, pendingBackupMusic, arg.AfterPath, arg.MaxRows)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var items []PendingBackupMusicRow
+	for rows.Next() {
+		var i PendingBackupMusicRow
+		if err := rows.Scan(
+			&i.ID,
+			&i.FilePath,
+			&i.FileSize,
+			&i.SentSize,
+			&i.SentMtime,
+			&i.SentError,
 		); err != nil {
 			return nil, err
 		}
@@ -157,6 +264,68 @@ func (q *Queries) PruneBackupRuns(ctx context.Context, startedAt pgtype.Timestam
 	return result.RowsAffected(), nil
 }
 
+const recordBackupMusic = `-- name: RecordBackupMusic :exec
+INSERT INTO backup_music_files (track_id, size, mtime, location, error)
+VALUES ($1, $2, $3, $4, $5)
+ON CONFLICT (track_id) DO UPDATE
+   SET size = EXCLUDED.size, mtime = EXCLUDED.mtime, location = EXCLUDED.location,
+       error = EXCLUDED.error, checked_at = now()
+`
+
+type RecordBackupMusicParams struct {
+	TrackID  pgtype.UUID
+	Size     int64
+	Mtime    pgtype.Timestamptz
+	Location string
+	Error    string
+}
+
+func (q *Queries) RecordBackupMusic(ctx context.Context, arg RecordBackupMusicParams) error {
+	_, err := q.db.Exec(ctx, recordBackupMusic,
+		arg.TrackID,
+		arg.Size,
+		arg.Mtime,
+		arg.Location,
+		arg.Error,
+	)
+	return err
+}
+
+const refusedBackupMusic = `-- name: RefusedBackupMusic :many
+SELECT t.file_path, m.error, m.checked_at
+  FROM backup_music_files m
+  JOIN tracks t ON t.id = m.track_id
+ WHERE m.error <> '' AND m.size = t.file_size AND t.updated_at <= m.checked_at
+ ORDER BY m.checked_at DESC
+ LIMIT $1
+`
+
+type RefusedBackupMusicRow struct {
+	FilePath  string
+	Error     string
+	CheckedAt pgtype.Timestamptz
+}
+
+func (q *Queries) RefusedBackupMusic(ctx context.Context, limit int32) ([]RefusedBackupMusicRow, error) {
+	rows, err := q.db.Query(ctx, refusedBackupMusic, limit)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var items []RefusedBackupMusicRow
+	for rows.Next() {
+		var i RefusedBackupMusicRow
+		if err := rows.Scan(&i.FilePath, &i.Error, &i.CheckedAt); err != nil {
+			return nil, err
+		}
+		items = append(items, i)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return items, nil
+}
+
 const saveBackupSettings = `-- name: SaveBackupSettings :exec
 UPDATE backup_settings
    SET enabled     = $1,
@@ -168,6 +337,7 @@ UPDATE backup_settings
        secret_key  = $7,
        run_hour    = $8,
        upload_limit_kib = $9,
+       include_music = $10,
        updated_at  = now()
  WHERE id = true
 `
@@ -182,6 +352,7 @@ type SaveBackupSettingsParams struct {
 	SecretKey      string
 	RunHour        int16
 	UploadLimitKib int32
+	IncludeMusic   bool
 }
 
 func (q *Queries) SaveBackupSettings(ctx context.Context, arg SaveBackupSettingsParams) error {
@@ -195,6 +366,7 @@ func (q *Queries) SaveBackupSettings(ctx context.Context, arg SaveBackupSettings
 		arg.SecretKey,
 		arg.RunHour,
 		arg.UploadLimitKib,
+		arg.IncludeMusic,
 	)
 	return err
 }

internal/db/dbq/models.go

@@ -305,6 +305,15 @@ type AuditLog struct {
 	CreatedAt pgtype.Timestamptz
 }
 
+type BackupMusicFile struct {
+	TrackID   pgtype.UUID
+	Size      int64
+	Mtime     pgtype.Timestamptz
+	Location  string
+	CheckedAt pgtype.Timestamptz
+	Error     string
+}
+
 type BackupRun struct {
 	ID         pgtype.UUID
 	Trigger    string
@@ -314,6 +323,9 @@ type BackupRun struct {
 	Location   string
 	Bytes      int64
 	Error      string
+	MusicFiles int32
+	MusicBytes int64
+	MusicNote  string
 }
 
 type BackupSetting struct {
@@ -328,6 +340,7 @@ type BackupSetting struct {
 	RunHour        int16
 	UploadLimitKib int32
 	UpdatedAt      pgtype.Timestamptz
+	IncludeMusic   bool
 }
 
 type CandidateArtistTag struct {

internal/db/migrations/0100_backup_music.down.sql

@@ -0,0 +1,6 @@
+DROP TABLE IF EXISTS backup_music_files;
+ALTER TABLE backup_runs
+    DROP COLUMN IF EXISTS music_note,
+    DROP COLUMN IF EXISTS music_bytes,
+    DROP COLUMN IF EXISTS music_files;
+ALTER TABLE backup_settings DROP COLUMN IF EXISTS include_music;

internal/db/migrations/0100_backup_music.up.sql

@@ -0,0 +1,36 @@
+-- 0100_backup_music.up.sql — the music files in Minstrel's own backup.
+--
+-- Until now a backup held the database only. With include_music on, each
+-- library file is also sent once to music/<path> in the same destination,
+-- and sent again only when it changes.
+--
+--   backup_settings.include_music  off until an admin switches it on.
+--   backup_music_files             one row per track sent (or refused): the
+--                                  file's size and mtime when it was sent, so
+--                                  a later night sends only what is new or
+--                                  changed. A track the scan has touched since
+--                                  checked_at is looked at again; it goes up
+--                                  again only if the file itself changed, so a
+--                                  full rescan doesn't resend the library. A
+--                                  refusal is kept with its reason and tried
+--                                  again once the file changes.
+--   backup_runs.music_*            how much a run sent, and why it stopped.
+ALTER TABLE backup_settings ADD COLUMN include_music boolean NOT NULL DEFAULT false;
+
+ALTER TABLE backup_runs
+    ADD COLUMN music_files integer NOT NULL DEFAULT 0,
+    ADD COLUMN music_bytes bigint NOT NULL DEFAULT 0,
+    ADD COLUMN music_note  text NOT NULL DEFAULT '';
+
+CREATE TABLE backup_music_files (
+    track_id uuid PRIMARY KEY REFERENCES tracks (id) ON DELETE CASCADE,
+    -- The file as it was sent.
+    size       bigint NOT NULL,
+    mtime      timestamptz NOT NULL,
+    -- Where it went: the object key, or the file's path in the folder.
+    location   text NOT NULL,
+    -- When the file was last sent or found unchanged.
+    checked_at timestamptz NOT NULL DEFAULT now(),
+    -- Empty when it was sent; otherwise why the destination refused it.
+    error    text NOT NULL DEFAULT ''
+);

internal/db/queries/backups.sql

@@ -14,6 +14,7 @@ UPDATE backup_settings
        secret_key  = $7,
        run_hour    = $8,
        upload_limit_kib = $9,
+       include_music = $10,
        updated_at  = now()
  WHERE id = true;
 
@@ -27,11 +28,15 @@ UPDATE backup_runs
        finished_at = now(),
        location    = $3,
        bytes       = $4,
-       error       = $5
+       error       = $5,
+       music_files = $6,
+       music_bytes = $7,
+       music_note  = $8
  WHERE id = $1;
 
 -- name: ListBackupRuns :many
-SELECT id, trigger, status, started_at, finished_at, location, bytes, error
+SELECT id, trigger, status, started_at, finished_at, location, bytes, error,
+       music_files, music_bytes, music_note
   FROM backup_runs
  ORDER BY started_at DESC
  LIMIT $1;
@@ -59,3 +64,47 @@ UPDATE backup_runs
 
 -- name: PruneBackupRuns :execrows
 DELETE FROM backup_runs WHERE status <> 'running' AND started_at < $1;
+
+-- name: PendingBackupMusic :many
+-- Library files the backup may not hold as they are now, in path order from
+-- after_path, so a night that runs out of time carries on where it stopped:
+-- never sent, a different size, touched by a scan since they were last
+-- checked, or refused before the settings were last saved (a fixed key gets
+-- another go). The caller compares the file's own size and mtime before
+-- sending.
+SELECT t.id, t.file_path, t.file_size,
+       m.size AS sent_size, m.mtime AS sent_mtime, m.error AS sent_error
+  FROM tracks t
+  LEFT JOIN backup_music_files m ON m.track_id = t.id
+ WHERE t.missing_since IS NULL
+   AND t.file_path > sqlc.arg(after_path)::text
+   AND (m.track_id IS NULL OR m.size <> t.file_size OR t.updated_at > m.checked_at
+        OR (m.error <> '' AND m.checked_at < (SELECT updated_at FROM backup_settings)))
+ ORDER BY t.file_path
+ LIMIT sqlc.arg(max_rows);
+
+-- name: RecordBackupMusic :exec
+INSERT INTO backup_music_files (track_id, size, mtime, location, error)
+VALUES ($1, $2, $3, $4, $5)
+ON CONFLICT (track_id) DO UPDATE
+   SET size = EXCLUDED.size, mtime = EXCLUDED.mtime, location = EXCLUDED.location,
+       error = EXCLUDED.error, checked_at = now();
+
+-- name: BackupMusicProgress :one
+-- How much of the library the backup holds as it is now.
+SELECT count(*)::bigint AS total_files,
+       COALESCE(sum(t.file_size), 0)::bigint AS total_bytes,
+       count(*) FILTER (WHERE m.error = '' AND m.size = t.file_size AND t.updated_at <= m.checked_at)::bigint AS sent_files,
+       COALESCE(sum(t.file_size) FILTER (WHERE m.error = '' AND m.size = t.file_size AND t.updated_at <= m.checked_at), 0)::bigint AS sent_bytes,
+       count(*) FILTER (WHERE m.error <> '' AND m.size = t.file_size AND t.updated_at <= m.checked_at)::bigint AS refused_files
+  FROM tracks t
+  LEFT JOIN backup_music_files m ON m.track_id = t.id
+ WHERE t.missing_since IS NULL;
+
+-- name: RefusedBackupMusic :many
+SELECT t.file_path, m.error, m.checked_at
+  FROM backup_music_files m
+  JOIN tracks t ON t.id = m.track_id
+ WHERE m.error <> '' AND m.size = t.file_size AND t.updated_at <= m.checked_at
+ ORDER BY m.checked_at DESC
+ LIMIT $1;