minstrel/internal/backup/music.go

package backup

import (
	"context"
	"crypto/sha256"
	"encoding/hex"
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/jackc/pgx/v5/pgtype"

	"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
)

const (
	// The music's folder inside the destination, beside database/.
	musicDir = "music"
	// MusicBudget is how long one run may spend sending music. A library
	// bigger than a night's worth carries on from where it stopped on the
	// next run, so the first copy of a large library spans several nights.
	MusicBudget = 6 * time.Hour
	musicBatch  = 200
	// refusedInARow stops a pass: one refusal is that file, several in a row
	// are the key or the bucket refusing the music as a whole.
	refusedInARow = 3
)

// MusicProgress is how much of the library the backup holds as it is now.
type MusicProgress struct {
	TotalFiles   int64         `json:"total_files"`
	TotalBytes   int64         `json:"total_bytes"`
	SentFiles    int64         `json:"sent_files"`
	SentBytes    int64         `json:"sent_bytes"`
	RefusedFiles int64         `json:"refused_files"`
	Refused      []RefusedFile `json:"refused"`
}

// RefusedFile is a library file the destination would not take, with its
// answer. It is tried again once the file changes.
type RefusedFile struct {
	Path  string    `json:"path"`
	Error string    `json:"error"`
	At    time.Time `json:"at"`
}

// musicSent is what one run's music pass did.
type musicSent struct {
	files int
	bytes int64
	note  string
}

// commonRoot is the deepest folder holding every library root, so the music's
// paths in the backup are the same whichever root a file is under.
func commonRoot(roots []string) string {
	if len(roots) == 0 {
		return ""
	}
	common := strings.Split(filepath.Clean(roots[0]), string(filepath.Separator))
	for _, r := range roots[1:] {
		parts := strings.Split(filepath.Clean(r), string(filepath.Separator))
		n := 0
		for n < len(common) && n < len(parts) && common[n] == parts[n] {
			n++
		}
		common = common[:n]
	}
	root := strings.Join(common, string(filepath.Separator))
	if root == "" {
		return string(filepath.Separator)
	}
	return root
}

// musicPath is file's path inside music/, relative to root, with forward
// slashes. A file outside root keeps its whole path.
func musicPath(root, file string) string {
	rel, err := filepath.Rel(root, file)
	if root == "" || err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
		rel = strings.TrimPrefix(filepath.Clean(file), string(filepath.Separator))
	}
	return musicDir + "/" + filepath.ToSlash(rel)
}

func (s *Service) musicProgress(ctx context.Context) (*MusicProgress, error) {
	q := dbq.New(s.pool)
	p, err := q.BackupMusicProgress(ctx)
	if err != nil {
		return nil, fmt.Errorf("backup: read music progress: %w", err)
	}
	refused, err := q.RefusedBackupMusic(ctx, recentRuns)
	if err != nil {
		return nil, fmt.Errorf("backup: list refused music: %w", err)
	}
	out := &MusicProgress{
		TotalFiles: p.TotalFiles, TotalBytes: p.TotalBytes,
		SentFiles: p.SentFiles, SentBytes: p.SentBytes,
		RefusedFiles: p.RefusedFiles, Refused: make([]RefusedFile, 0, len(refused)),
	}
	for _, r := range refused {
		out.Refused = append(out.Refused, RefusedFile{Path: r.FilePath, Error: r.Error, At: r.CheckedAt.Time})
	}
	return out, nil
}

// sendMusic sends library files the destination doesn't hold as they are now,
// one at a time in path order, until all are sent or ctx ends. Every file is
// recorded as it lands, so a run cut short loses nothing. It never deletes
// anything at the destination.
func (s *Service) sendMusic(ctx context.Context, st dbq.BackupSetting) musicSent {
	var out musicSent
	q := dbq.New(s.pool)
	var roots []string
	if s.Roots != nil {
		roots = s.Roots()
	}
	root := commonRoot(roots)
	var bucket Bucket
	if st.Destination == DestinationS3 {
		b, err := bucketOf(st)
		if err != nil {
			out.note = describe(err)
			return out
		}
		bucket = b
	}

	after := ""
	refused := 0
	for {
		rows, err := q.PendingBackupMusic(ctx, dbq.PendingBackupMusicParams{AfterPath: after, MaxRows: musicBatch})
		if err != nil {
			out.note = musicStopped(ctx, err)
			return out
		}
		if len(rows) == 0 {
			return out
		}
		for _, r := range rows {
			after = r.FilePath
			if err := ctx.Err(); err != nil {
				out.note = musicStopped(ctx, err)
				return out
			}
			info, err := os.Stat(r.FilePath)
			if err != nil || !info.Mode().IsRegular() {
				// Gone or unreadable: the library scan settles that, and the
				// file is looked at again on the next run.
				continue
			}
			mtime := info.ModTime().Truncate(time.Microsecond)
			key := musicPath(root, r.FilePath)
			location := key
			if st.Destination == DestinationFolder {
				location = filepath.Join(st.Folder, filepath.FromSlash(key))
			}
			rec := dbq.RecordBackupMusicParams{
				TrackID: r.ID, Size: info.Size(),
				Mtime:    pgtype.Timestamptz{Time: mtime, Valid: true},
				Location: location,
			}
			if r.SentSize != nil && *r.SentSize == info.Size() && r.SentMtime.Valid && r.SentMtime.Time.Equal(mtime) &&
				(r.SentError == nil || *r.SentError == "") {
				// Touched by a scan but the same file: nothing to send.
				if err := q.RecordBackupMusic(ctx, rec); err != nil {
					out.note = musicStopped(ctx, err)
					return out
				}
				continue
			}

			if st.Destination == DestinationFolder {
				err = copyFile(r.FilePath, location, mtime)
			} else {
				err = putFile(ctx, bucket, key, r.FilePath, int64(st.UploadLimitKib)*1024)
			}
			switch {
			case err == nil:
				out.files++
				out.bytes += info.Size()
				refused = 0
			case ObjectRefused(err):
				rec.Error = strings.TrimPrefix(err.Error(), "backup: ")
				if refused++; refused >= refusedInARow {
					out.note = fmt.Sprintf("Music stopped: the destination refused %d files in a row (%s). "+
						"Check the key may write under %s/; refused files go again once you save the settings.",
						refused, rec.Error, musicDir)
				}
			case ctx.Err() == nil && errors.Is(err, errSourceRead):
				// The library file couldn't be read through: looked at again next run.
				continue
			default:
				out.note = musicStopped(ctx, err)
				return out
			}
			if err := q.RecordBackupMusic(ctx, rec); err != nil {
				out.note = musicStopped(ctx, err)
				return out
			}
			if out.note != "" {
				return out
			}
		}
	}
}

// musicStopped is why a music pass ended early, as the panel shows it.
func musicStopped(ctx context.Context, err error) string {
	switch {
	case errors.Is(ctx.Err(), context.DeadlineExceeded):
		return fmt.Sprintf("Music stopped for tonight after %d hours; it carries on with the next backup.", int(MusicBudget/time.Hour))
	case ctx.Err() != nil:
		return "Music stopped because the server stopped; it carries on with the next backup."
	}
	return "Music stopped: " + describe(err) + " It carries on with the next backup."
}

// errSourceRead marks a library file that could not be read.
var errSourceRead = errors.New("backup: the library file could not be read")

// putFile uploads file to key. It reads the file twice, once to hash it
// (S3 signs over the body's hash) and once to send it through the limit.
func putFile(ctx context.Context, b Bucket, key, file string, perSecond int64) error {
	f, err := os.Open(file)
	if err != nil {
		return fmt.Errorf("%w: %v", errSourceRead, err)
	}
	defer func() { _ = f.Close() }()
	h := sha256.New()
	size, err := io.Copy(h, f)
	if err != nil {
		return fmt.Errorf("%w: %v", errSourceRead, err)
	}
	if _, err := f.Seek(0, io.SeekStart); err != nil {
		return fmt.Errorf("%w: %v", errSourceRead, err)
	}
	return b.Put(ctx, key, Throttle(ctx, f, perSecond), size, hex.EncodeToString(h.Sum(nil)))
}

// copyFile copies file to dst under a hidden .part name first, so the folder
// never holds half a file under the file's own name. The copy keeps the
// file's modification time. The upload limit is the bucket's: a folder copy
// runs at the disk's pace.
func copyFile(file, dst string, mtime time.Time) error {
	in, err := os.Open(file)
	if err != nil {
		return fmt.Errorf("%w: %v", errSourceRead, err)
	}
	defer func() { _ = in.Close() }()
	if err := os.MkdirAll(filepath.Dir(dst), 0o700); err != nil {
		return fmt.Errorf("backup: create %s: %w", filepath.Dir(dst), err)
	}
	part := filepath.Join(filepath.Dir(dst), "."+filepath.Base(dst)+".part")
	out, err := os.OpenFile(part, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600)
	if err != nil {
		return fmt.Errorf("backup: create %s: %w", part, err)
	}
	_, err = io.Copy(out, in)
	if err == nil {
		err = out.Sync()
	}
	if cerr := out.Close(); err == nil {
		err = cerr
	}
	if err == nil {
		err = os.Chtimes(part, mtime, mtime)
	}
	if err == nil {
		err = os.Rename(part, dst)
	}
	if err != nil {
		_ = os.Remove(part)
		return fmt.Errorf("backup: write %s: %w", dst, err)
	}
	return nil
}