package backup import ( "context" "crypto/sha256" "encoding/hex" "errors" "fmt" "io" "os" "path/filepath" "strings" "time" "github.com/jackc/pgx/v5/pgtype" "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" ) const ( // The music's folder inside the destination, beside database/. musicDir = "music" // MusicBudget is how long one run may spend sending music. A library // bigger than a night's worth carries on from where it stopped on the // next run, so the first copy of a large library spans several nights. MusicBudget = 6 * time.Hour musicBatch = 200 // refusedInARow stops a pass: one refusal is that file, several in a row // are the key or the bucket refusing the music as a whole. refusedInARow = 3 ) // MusicProgress is how much of the library the backup holds as it is now. type MusicProgress struct { TotalFiles int64 `json:"total_files"` TotalBytes int64 `json:"total_bytes"` SentFiles int64 `json:"sent_files"` SentBytes int64 `json:"sent_bytes"` RefusedFiles int64 `json:"refused_files"` Refused []RefusedFile `json:"refused"` } // RefusedFile is a library file the destination would not take, with its // answer. It is tried again once the file changes. type RefusedFile struct { Path string `json:"path"` Error string `json:"error"` At time.Time `json:"at"` } // musicSent is what one run's music pass did. type musicSent struct { files int bytes int64 note string } // commonRoot is the deepest folder holding every library root, so the music's // paths in the backup are the same whichever root a file is under. func commonRoot(roots []string) string { if len(roots) == 0 { return "" } common := strings.Split(filepath.Clean(roots[0]), string(filepath.Separator)) for _, r := range roots[1:] { parts := strings.Split(filepath.Clean(r), string(filepath.Separator)) n := 0 for n < len(common) && n < len(parts) && common[n] == parts[n] { n++ } common = common[:n] } root := strings.Join(common, string(filepath.Separator)) if root == "" { return string(filepath.Separator) } return root } // musicPath is file's path inside music/, relative to root, with forward // slashes. A file outside root keeps its whole path. func musicPath(root, file string) string { rel, err := filepath.Rel(root, file) if root == "" || err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { rel = strings.TrimPrefix(filepath.Clean(file), string(filepath.Separator)) } return musicDir + "/" + filepath.ToSlash(rel) } func (s *Service) musicProgress(ctx context.Context) (*MusicProgress, error) { q := dbq.New(s.pool) p, err := q.BackupMusicProgress(ctx) if err != nil { return nil, fmt.Errorf("backup: read music progress: %w", err) } refused, err := q.RefusedBackupMusic(ctx, recentRuns) if err != nil { return nil, fmt.Errorf("backup: list refused music: %w", err) } out := &MusicProgress{ TotalFiles: p.TotalFiles, TotalBytes: p.TotalBytes, SentFiles: p.SentFiles, SentBytes: p.SentBytes, RefusedFiles: p.RefusedFiles, Refused: make([]RefusedFile, 0, len(refused)), } for _, r := range refused { out.Refused = append(out.Refused, RefusedFile{Path: r.FilePath, Error: r.Error, At: r.CheckedAt.Time}) } return out, nil } // sendMusic sends library files the destination doesn't hold as they are now, // one at a time in path order, until all are sent or ctx ends. Every file is // recorded as it lands, so a run cut short loses nothing. It never deletes // anything at the destination. func (s *Service) sendMusic(ctx context.Context, st dbq.BackupSetting) musicSent { var out musicSent q := dbq.New(s.pool) var roots []string if s.Roots != nil { roots = s.Roots() } root := commonRoot(roots) var bucket Bucket if st.Destination == DestinationS3 { b, err := bucketOf(st) if err != nil { out.note = describe(err) return out } bucket = b } after := "" refused := 0 for { rows, err := q.PendingBackupMusic(ctx, dbq.PendingBackupMusicParams{AfterPath: after, MaxRows: musicBatch}) if err != nil { out.note = musicStopped(ctx, err) return out } if len(rows) == 0 { return out } for _, r := range rows { after = r.FilePath if err := ctx.Err(); err != nil { out.note = musicStopped(ctx, err) return out } info, err := os.Stat(r.FilePath) if err != nil || !info.Mode().IsRegular() { // Gone or unreadable: the library scan settles that, and the // file is looked at again on the next run. continue } mtime := info.ModTime().Truncate(time.Microsecond) key := musicPath(root, r.FilePath) location := key if st.Destination == DestinationFolder { location = filepath.Join(st.Folder, filepath.FromSlash(key)) } rec := dbq.RecordBackupMusicParams{ TrackID: r.ID, Size: info.Size(), Mtime: pgtype.Timestamptz{Time: mtime, Valid: true}, Location: location, } if r.SentSize != nil && *r.SentSize == info.Size() && r.SentMtime.Valid && r.SentMtime.Time.Equal(mtime) && (r.SentError == nil || *r.SentError == "") { // Touched by a scan but the same file: nothing to send. if err := q.RecordBackupMusic(ctx, rec); err != nil { out.note = musicStopped(ctx, err) return out } continue } if st.Destination == DestinationFolder { err = copyFile(r.FilePath, location, mtime) } else { err = putFile(ctx, bucket, key, r.FilePath, int64(st.UploadLimitKib)*1024) } switch { case err == nil: out.files++ out.bytes += info.Size() refused = 0 case ObjectRefused(err): rec.Error = strings.TrimPrefix(err.Error(), "backup: ") if refused++; refused >= refusedInARow { out.note = fmt.Sprintf("Music stopped: the destination refused %d files in a row (%s). "+ "Check the key may write under %s/; refused files go again once you save the settings.", refused, rec.Error, musicDir) } case ctx.Err() == nil && errors.Is(err, errSourceRead): // The library file couldn't be read through: looked at again next run. continue default: out.note = musicStopped(ctx, err) return out } if err := q.RecordBackupMusic(ctx, rec); err != nil { out.note = musicStopped(ctx, err) return out } if out.note != "" { return out } } } } // musicStopped is why a music pass ended early, as the panel shows it. func musicStopped(ctx context.Context, err error) string { switch { case errors.Is(ctx.Err(), context.DeadlineExceeded): return fmt.Sprintf("Music stopped for tonight after %d hours; it carries on with the next backup.", int(MusicBudget/time.Hour)) case ctx.Err() != nil: return "Music stopped because the server stopped; it carries on with the next backup." } return "Music stopped: " + describe(err) + " It carries on with the next backup." } // errSourceRead marks a library file that could not be read. var errSourceRead = errors.New("backup: the library file could not be read") // putFile uploads file to key. It reads the file twice, once to hash it // (S3 signs over the body's hash) and once to send it through the limit. func putFile(ctx context.Context, b Bucket, key, file string, perSecond int64) error { f, err := os.Open(file) if err != nil { return fmt.Errorf("%w: %v", errSourceRead, err) } defer func() { _ = f.Close() }() h := sha256.New() size, err := io.Copy(h, f) if err != nil { return fmt.Errorf("%w: %v", errSourceRead, err) } if _, err := f.Seek(0, io.SeekStart); err != nil { return fmt.Errorf("%w: %v", errSourceRead, err) } return b.Put(ctx, key, Throttle(ctx, f, perSecond), size, hex.EncodeToString(h.Sum(nil))) } // copyFile copies file to dst under a hidden .part name first, so the folder // never holds half a file under the file's own name. The copy keeps the // file's modification time. The upload limit is the bucket's: a folder copy // runs at the disk's pace. func copyFile(file, dst string, mtime time.Time) error { in, err := os.Open(file) if err != nil { return fmt.Errorf("%w: %v", errSourceRead, err) } defer func() { _ = in.Close() }() if err := os.MkdirAll(filepath.Dir(dst), 0o700); err != nil { return fmt.Errorf("backup: create %s: %w", filepath.Dir(dst), err) } part := filepath.Join(filepath.Dir(dst), "."+filepath.Base(dst)+".part") out, err := os.OpenFile(part, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600) if err != nil { return fmt.Errorf("backup: create %s: %w", part, err) } _, err = io.Copy(out, in) if err == nil { err = out.Sync() } if cerr := out.Close(); err == nil { err = cerr } if err == nil { err = os.Chtimes(part, mtime, mtime) } if err == nil { err = os.Rename(part, dst) } if err != nil { _ = os.Remove(part) return fmt.Errorf("backup: write %s: %w", dst, err) } return nil }