feat: the backup can include the music, sending each library file once to music/ and again only when it changes (#6774)

With "Include the music" on, each night's run sends the library files the
destination doesn't hold yet, after the database file has landed, for up to
six hours; a large library spans several nights the first time. Files keep
their path under the library folders' common root, beside database/ in the
same folder or bucket.

backup_music_files records each file's size and mtime as sent, so a night
sends only new and changed files, and a rescan that touches a track but finds
the same file sends nothing. A changed file goes to its old name; Minstrel
never deletes there. An object the bucket refuses (a locked overwrite) is
kept with its answer and goes again once the file changes or the settings are
saved; three refusals in a row stop the night's music, since that is a key
that may not write under music/. A folder copy is written under a .part name
and renamed, keeping the file's time.

The panel shows how much of the library is in, the refused files, and each
run's music count and note. The verdict still turns on the database file.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/backup/music.go

@@ -0,0 +1,281 @@
+package backup
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/jackc/pgx/v5/pgtype"
+
+	"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
+)
+
+const (
+	// The music's folder inside the destination, beside database/.
+	musicDir = "music"
+	// MusicBudget is how long one run may spend sending music. A library
+	// bigger than a night's worth carries on from where it stopped on the
+	// next run, so the first copy of a large library spans several nights.
+	MusicBudget = 6 * time.Hour
+	musicBatch  = 200
+	// refusedInARow stops a pass: one refusal is that file, several in a row
+	// are the key or the bucket refusing the music as a whole.
+	refusedInARow = 3
+)
+
+// MusicProgress is how much of the library the backup holds as it is now.
+type MusicProgress struct {
+	TotalFiles   int64         `json:"total_files"`
+	TotalBytes   int64         `json:"total_bytes"`
+	SentFiles    int64         `json:"sent_files"`
+	SentBytes    int64         `json:"sent_bytes"`
+	RefusedFiles int64         `json:"refused_files"`
+	Refused      []RefusedFile `json:"refused"`
+}
+
+// RefusedFile is a library file the destination would not take, with its
+// answer. It is tried again once the file changes.
+type RefusedFile struct {
+	Path  string    `json:"path"`
+	Error string    `json:"error"`
+	At    time.Time `json:"at"`
+}
+
+// musicSent is what one run's music pass did.
+type musicSent struct {
+	files int
+	bytes int64
+	note  string
+}
+
+// commonRoot is the deepest folder holding every library root, so the music's
+// paths in the backup are the same whichever root a file is under.
+func commonRoot(roots []string) string {
+	if len(roots) == 0 {
+		return ""
+	}
+	common := strings.Split(filepath.Clean(roots[0]), string(filepath.Separator))
+	for _, r := range roots[1:] {
+		parts := strings.Split(filepath.Clean(r), string(filepath.Separator))
+		n := 0
+		for n < len(common) && n < len(parts) && common[n] == parts[n] {
+			n++
+		}
+		common = common[:n]
+	}
+	root := strings.Join(common, string(filepath.Separator))
+	if root == "" {
+		return string(filepath.Separator)
+	}
+	return root
+}
+
+// musicPath is file's path inside music/, relative to root, with forward
+// slashes. A file outside root keeps its whole path.
+func musicPath(root, file string) string {
+	rel, err := filepath.Rel(root, file)
+	if root == "" || err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
+		rel = strings.TrimPrefix(filepath.Clean(file), string(filepath.Separator))
+	}
+	return musicDir + "/" + filepath.ToSlash(rel)
+}
+
+func (s *Service) musicProgress(ctx context.Context) (*MusicProgress, error) {
+	q := dbq.New(s.pool)
+	p, err := q.BackupMusicProgress(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("backup: read music progress: %w", err)
+	}
+	refused, err := q.RefusedBackupMusic(ctx, recentRuns)
+	if err != nil {
+		return nil, fmt.Errorf("backup: list refused music: %w", err)
+	}
+	out := &MusicProgress{
+		TotalFiles: p.TotalFiles, TotalBytes: p.TotalBytes,
+		SentFiles: p.SentFiles, SentBytes: p.SentBytes,
+		RefusedFiles: p.RefusedFiles, Refused: make([]RefusedFile, 0, len(refused)),
+	}
+	for _, r := range refused {
+		out.Refused = append(out.Refused, RefusedFile{Path: r.FilePath, Error: r.Error, At: r.CheckedAt.Time})
+	}
+	return out, nil
+}
+
+// sendMusic sends library files the destination doesn't hold as they are now,
+// one at a time in path order, until all are sent or ctx ends. Every file is
+// recorded as it lands, so a run cut short loses nothing. It never deletes
+// anything at the destination.
+func (s *Service) sendMusic(ctx context.Context, st dbq.BackupSetting) musicSent {
+	var out musicSent
+	q := dbq.New(s.pool)
+	var roots []string
+	if s.Roots != nil {
+		roots = s.Roots()
+	}
+	root := commonRoot(roots)
+	var bucket Bucket
+	if st.Destination == DestinationS3 {
+		b, err := bucketOf(st)
+		if err != nil {
+			out.note = describe(err)
+			return out
+		}
+		bucket = b
+	}
+
+	after := ""
+	refused := 0
+	for {
+		rows, err := q.PendingBackupMusic(ctx, dbq.PendingBackupMusicParams{AfterPath: after, MaxRows: musicBatch})
+		if err != nil {
+			out.note = musicStopped(ctx, err)
+			return out
+		}
+		if len(rows) == 0 {
+			return out
+		}
+		for _, r := range rows {
+			after = r.FilePath
+			if err := ctx.Err(); err != nil {
+				out.note = musicStopped(ctx, err)
+				return out
+			}
+			info, err := os.Stat(r.FilePath)
+			if err != nil || !info.Mode().IsRegular() {
+				// Gone or unreadable: the library scan settles that, and the
+				// file is looked at again on the next run.
+				continue
+			}
+			mtime := info.ModTime().Truncate(time.Microsecond)
+			key := musicPath(root, r.FilePath)
+			location := key
+			if st.Destination == DestinationFolder {
+				location = filepath.Join(st.Folder, filepath.FromSlash(key))
+			}
+			rec := dbq.RecordBackupMusicParams{
+				TrackID: r.ID, Size: info.Size(),
+				Mtime:    pgtype.Timestamptz{Time: mtime, Valid: true},
+				Location: location,
+			}
+			if r.SentSize != nil && *r.SentSize == info.Size() && r.SentMtime.Valid && r.SentMtime.Time.Equal(mtime) &&
+				(r.SentError == nil || *r.SentError == "") {
+				// Touched by a scan but the same file: nothing to send.
+				if err := q.RecordBackupMusic(ctx, rec); err != nil {
+					out.note = musicStopped(ctx, err)
+					return out
+				}
+				continue
+			}
+
+			if st.Destination == DestinationFolder {
+				err = copyFile(r.FilePath, location, mtime)
+			} else {
+				err = putFile(ctx, bucket, key, r.FilePath, int64(st.UploadLimitKib)*1024)
+			}
+			switch {
+			case err == nil:
+				out.files++
+				out.bytes += info.Size()
+				refused = 0
+			case ObjectRefused(err):
+				rec.Error = strings.TrimPrefix(err.Error(), "backup: ")
+				if refused++; refused >= refusedInARow {
+					out.note = fmt.Sprintf("Music stopped: the destination refused %d files in a row (%s). "+
+						"Check the key may write under %s/; refused files go again once you save the settings.",
+						refused, rec.Error, musicDir)
+				}
+			case ctx.Err() == nil && errors.Is(err, errSourceRead):
+				// The library file couldn't be read through: looked at again next run.
+				continue
+			default:
+				out.note = musicStopped(ctx, err)
+				return out
+			}
+			if err := q.RecordBackupMusic(ctx, rec); err != nil {
+				out.note = musicStopped(ctx, err)
+				return out
+			}
+			if out.note != "" {
+				return out
+			}
+		}
+	}
+}
+
+// musicStopped is why a music pass ended early, as the panel shows it.
+func musicStopped(ctx context.Context, err error) string {
+	switch {
+	case errors.Is(ctx.Err(), context.DeadlineExceeded):
+		return fmt.Sprintf("Music stopped for tonight after %d hours; it carries on with the next backup.", int(MusicBudget/time.Hour))
+	case ctx.Err() != nil:
+		return "Music stopped because the server stopped; it carries on with the next backup."
+	}
+	return "Music stopped: " + describe(err) + " It carries on with the next backup."
+}
+
+// errSourceRead marks a library file that could not be read.
+var errSourceRead = errors.New("backup: the library file could not be read")
+
+// putFile uploads file to key. It reads the file twice, once to hash it
+// (S3 signs over the body's hash) and once to send it through the limit.
+func putFile(ctx context.Context, b Bucket, key, file string, perSecond int64) error {
+	f, err := os.Open(file)
+	if err != nil {
+		return fmt.Errorf("%w: %v", errSourceRead, err)
+	}
+	defer func() { _ = f.Close() }()
+	h := sha256.New()
+	size, err := io.Copy(h, f)
+	if err != nil {
+		return fmt.Errorf("%w: %v", errSourceRead, err)
+	}
+	if _, err := f.Seek(0, io.SeekStart); err != nil {
+		return fmt.Errorf("%w: %v", errSourceRead, err)
+	}
+	return b.Put(ctx, key, Throttle(ctx, f, perSecond), size, hex.EncodeToString(h.Sum(nil)))
+}
+
+// copyFile copies file to dst under a hidden .part name first, so the folder
+// never holds half a file under the file's own name. The copy keeps the
+// file's modification time. The upload limit is the bucket's: a folder copy
+// runs at the disk's pace.
+func copyFile(file, dst string, mtime time.Time) error {
+	in, err := os.Open(file)
+	if err != nil {
+		return fmt.Errorf("%w: %v", errSourceRead, err)
+	}
+	defer func() { _ = in.Close() }()
+	if err := os.MkdirAll(filepath.Dir(dst), 0o700); err != nil {
+		return fmt.Errorf("backup: create %s: %w", filepath.Dir(dst), err)
+	}
+	part := filepath.Join(filepath.Dir(dst), "."+filepath.Base(dst)+".part")
+	out, err := os.OpenFile(part, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600)
+	if err != nil {
+		return fmt.Errorf("backup: create %s: %w", part, err)
+	}
+	_, err = io.Copy(out, in)
+	if err == nil {
+		err = out.Sync()
+	}
+	if cerr := out.Close(); err == nil {
+		err = cerr
+	}
+	if err == nil {
+		err = os.Chtimes(part, mtime, mtime)
+	}
+	if err == nil {
+		err = os.Rename(part, dst)
+	}
+	if err != nil {
+		_ = os.Remove(part)
+		return fmt.Errorf("backup: write %s: %w", dst, err)
+	}
+	return nil
+}