package backup import ( "context" "crypto/sha256" "encoding/hex" "errors" "fmt" "io" "log/slog" "net/url" "os" "path" "path/filepath" "strings" "sync" "time" "github.com/jackc/pgx/v5/pgtype" "github.com/jackc/pgx/v5/pgxpool" "git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq" "git.fabledsword.com/bvandeusen/minstrel/internal/secret" ) // Where a backup can go. const ( DestinationFolder = "folder" DestinationS3 = "s3" ) // What started a run. const ( TriggerSchedule = "schedule" TriggerManual = "manual" ) // The verdict a Status gives, worst first in the order an admin acts on them. const ( StateOff = "off" // the nightly backup is switched off StateIncomplete = "incomplete" // switched on, but the destination isn't set up StateFailing = "failing" // the latest run failed StateWaiting = "waiting" // set up, and no backup has finished yet StateOverdue = "overdue" // the latest good backup is more than a day old StateOK = "ok" ) const ( // RunLimit bounds one run, snapshot and upload together. The web panel // stops polling a run a little after this, since the run is over by then. RunLimit = 30 * time.Minute // A nightly backup that hasn't succeeded for this long is overdue; the // slack past a day covers a run that starts late or takes a while. overdueAfter = 26 * time.Hour tickEvery = 5 * time.Minute // A failed run is tried again after at least this long, and after longer // each time it fails again, so a destination that is down isn't hammered. retryAfter = time.Hour recentRuns = 10 // Run history older than this is pruned after each run. keepRunsFor = 90 * 24 * time.Hour testTimeout = 30 * time.Second // The database backups' folder inside the destination. The music, when // it is backed up, sits beside it in the same destination. databaseDir = "database" ) var ( // ErrRunning is returned by Start while a backup is running. ErrRunning = errors.New("backup: a backup is already running") // ErrNotReady is returned by Start when the destination isn't set up. ErrNotReady = errors.New("backup: the destination isn't set up yet") // ErrClosed is returned by Start once the server is shutting down. ErrClosed = errors.New("backup: the server is shutting down") ) // FieldError is a setting that can't be saved, with the reason in words. type FieldError struct { Field string `json:"field"` Message string `json:"message"` } func (e *FieldError) Error() string { return e.Field + ": " + e.Message } // Config is the backup's settings as an admin sees them. The bucket's secret // key is never sent back, only whether one is stored. type Config struct { Enabled bool `json:"enabled"` Destination string `json:"destination"` Folder string `json:"folder"` Endpoint string `json:"endpoint"` Bucket string `json:"bucket"` AccessKey string `json:"access_key"` HasSecretKey bool `json:"has_secret_key"` RunHour int `json:"run_hour"` UploadLimitKiB int `json:"upload_limit_kib"` IncludeMusic bool `json:"include_music"` } // Update is a save from the admin card. An empty SecretKey keeps the stored // one, so the key doesn't have to be typed again on every save. type Update struct { Enabled bool `json:"enabled"` Destination string `json:"destination"` Folder string `json:"folder"` Endpoint string `json:"endpoint"` Bucket string `json:"bucket"` AccessKey string `json:"access_key"` SecretKey string `json:"secret_key"` RunHour int `json:"run_hour"` UploadLimitKiB int `json:"upload_limit_kib"` IncludeMusic bool `json:"include_music"` } // Run is one attempt at a backup. type Run struct { ID string `json:"id"` Trigger string `json:"trigger"` Status string `json:"status"` StartedAt time.Time `json:"started_at"` FinishedAt *time.Time `json:"finished_at"` Location string `json:"location"` Bytes int64 `json:"bytes"` Error string `json:"error"` // What the run sent of the music, and why that stopped early, if it did. MusicFiles int `json:"music_files"` MusicBytes int64 `json:"music_bytes"` MusicNote string `json:"music_note"` } // Status is everything the admin panel shows: a verdict, what stands in its // way, and the recent runs. type Status struct { Config Config `json:"config"` State string `json:"state"` Problems []string `json:"problems"` RunningSince *time.Time `json:"running_since"` LastSuccess *time.Time `json:"last_success"` FailingSince *time.Time `json:"failing_since"` // NextRun is when the schedule starts the next backup, or nil when it // won't: switched off, or not set up. NextRun *time.Time `json:"next_run"` RunLimitSeconds int `json:"run_limit_seconds"` Runs []Run `json:"runs"` // Music is how much of the library the backup holds, when it includes // the music. Music *MusicProgress `json:"music"` } // TestResult is what a destination test found: how many backups are there // already and when the newest was made. type TestResult struct { Backups int `json:"backups"` Latest *time.Time `json:"latest"` } // Service runs Minstrel's own backup: nightly when switched on, or when an // admin asks. One run at a time; each is recorded in backup_runs. type Service struct { pool *pgxpool.Pool logger *slog.Logger build string now func() time.Time // Roots lists the library's folders. The music's paths in the backup are // relative to the folder holding them all. Roots func() []string base context.Context cancel context.CancelFunc wg sync.WaitGroup mu sync.Mutex runningSince time.Time closed bool } // NewService returns a Service over pool. build is the server's version, // written into each backup's manifest. func NewService(pool *pgxpool.Pool, logger *slog.Logger, build string) *Service { base, cancel := context.WithCancel(context.Background()) return &Service{pool: pool, logger: logger, build: build, now: time.Now, base: base, cancel: cancel} } func (s *Service) settings(ctx context.Context) (dbq.BackupSetting, error) { st, err := dbq.New(s.pool).GetBackupSettings(ctx) if err != nil { return st, fmt.Errorf("backup: read settings: %w", err) } return st, nil } func view(st dbq.BackupSetting) Config { return Config{ Enabled: st.Enabled, Destination: st.Destination, Folder: st.Folder, Endpoint: st.Endpoint, Bucket: st.Bucket, AccessKey: st.AccessKey, HasSecretKey: st.SecretKey != "", RunHour: int(st.RunHour), UploadLimitKiB: int(st.UploadLimitKib), IncludeMusic: st.IncludeMusic, } } // Config returns the settings as an admin sees them. func (s *Service) Config(ctx context.Context) (Config, error) { st, err := s.settings(ctx) return view(st), err } // merge applies u over the stored settings: it checks each field, and seals // a new secret key. It returns a *FieldError for a value that can't be kept. func merge(st dbq.BackupSetting, u Update) (dbq.BackupSetting, error) { if u.Destination != DestinationFolder && u.Destination != DestinationS3 { return st, &FieldError{"destination", "Choose a folder or an S3 bucket."} } if u.RunHour < 0 || u.RunHour > 23 { return st, &FieldError{"run_hour", "The hour is from 0 to 23."} } if u.UploadLimitKiB < 0 || u.UploadLimitKiB > 1<<30 { return st, &FieldError{"upload_limit_kib", "The upload limit is 0 (no limit) or more."} } folder := strings.TrimSpace(u.Folder) if folder != "" && !filepath.IsAbs(folder) { return st, &FieldError{"folder", "Give the folder's full path, starting with /."} } endpoint := strings.TrimRight(strings.TrimSpace(u.Endpoint), "/") if endpoint != "" { e, err := url.Parse(endpoint) if err != nil || (e.Scheme != "http" && e.Scheme != "https") || e.Host == "" { return st, &FieldError{"endpoint", "Give the endpoint as a URL, such as https://s3.example.com."} } } st.Enabled = u.Enabled st.Destination = u.Destination st.Folder = filepath.Clean(folder) if folder == "" { st.Folder = "" } st.Endpoint = endpoint st.Bucket = strings.TrimSpace(u.Bucket) st.AccessKey = strings.TrimSpace(u.AccessKey) st.RunHour = int16(u.RunHour) st.UploadLimitKib = int32(u.UploadLimitKiB) st.IncludeMusic = u.IncludeMusic if u.SecretKey != "" { sealed, err := secret.Seal(secret.BackupSecretKey.Purpose(), "", u.SecretKey) if err != nil { return st, err } st.SecretKey = sealed } return st, nil } // Save stores u. A value that can't be kept is a *FieldError and nothing is // stored. Settings that are incomplete still save; Status lists what's missing. func (s *Service) Save(ctx context.Context, u Update) error { st, err := s.settings(ctx) if err != nil { return err } st, err = merge(st, u) if err != nil { return err } return dbq.New(s.pool).SaveBackupSettings(ctx, dbq.SaveBackupSettingsParams{ Enabled: st.Enabled, Destination: st.Destination, Folder: st.Folder, Endpoint: st.Endpoint, Bucket: st.Bucket, AccessKey: st.AccessKey, SecretKey: st.SecretKey, RunHour: st.RunHour, UploadLimitKib: st.UploadLimitKib, IncludeMusic: st.IncludeMusic, }) } // problems is what stops a backup from running with these settings, in words. func problems(st dbq.BackupSetting) []string { var out []string switch st.Destination { case DestinationFolder: if st.Folder == "" { out = append(out, "Choose the folder backups are written to.") } case DestinationS3: if st.Endpoint == "" { out = append(out, "Enter the bucket's endpoint.") } if st.Bucket == "" { out = append(out, "Enter the bucket's name.") } if st.AccessKey == "" { out = append(out, "Enter the access key.") } if st.SecretKey == "" { out = append(out, "Enter the secret key.") } default: out = append(out, "Choose where backups go.") } return out } func timeOf(t pgtype.Timestamptz) time.Time { if !t.Valid || t.Time.Unix() <= 0 { return time.Time{} } return t.Time } func ptr(t time.Time) *time.Time { if t.IsZero() { return nil } return &t } // runTimes is what the verdict and the schedule turn on. A zero time is a run // that never happened. type runTimes struct { lastSuccess, lastAttempt, failingSince time.Time } func (s *Service) times(ctx context.Context) (runTimes, error) { row, err := dbq.New(s.pool).BackupRunTimes(ctx) if err != nil { return runTimes{}, fmt.Errorf("backup: read run times: %w", err) } return runTimes{timeOf(row.LastSuccess), timeOf(row.LastAttempt), timeOf(row.FailingSince)}, nil } // slotBefore is the latest scheduled start at or before now: today at hour // UTC, or yesterday's if that hour hasn't come yet. func slotBefore(now time.Time, hour int) time.Time { now = now.UTC() slot := time.Date(now.Year(), now.Month(), now.Day(), hour, 0, 0, 0, time.UTC) if slot.After(now) { slot = slot.AddDate(0, 0, -1) } return slot } // due reports when the schedule next starts a backup: now or earlier means // the next tick starts one. A slot that passed with no attempt since is due, // so a server that was down at the hour catches up. A run that failed is // tried again, waiting at least retryAfter and then as long again as it has // been failing, until it succeeds or the next slot comes. func due(now time.Time, hour int, t runTimes) time.Time { slot := slotBefore(now, hour) if t.lastAttempt.Before(slot) { return slot } next := slot.AddDate(0, 0, 1) if !t.failingSince.IsZero() { wait := max(retryAfter, t.lastAttempt.Sub(t.failingSince)) if retry := t.lastAttempt.Add(wait); retry.Before(next) { return retry } } return next } func verdict(st dbq.BackupSetting, probs []string, t runTimes, now time.Time) string { switch { case !st.Enabled: return StateOff case len(probs) > 0: return StateIncomplete case !t.failingSince.IsZero(): return StateFailing case t.lastSuccess.IsZero(): return StateWaiting case now.Sub(t.lastSuccess) > overdueAfter: return StateOverdue } return StateOK } func uuidText(u pgtype.UUID) string { b := u.Bytes return fmt.Sprintf("%x-%x-%x-%x-%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:16]) } // Status is the verdict, what stands in its way, and the recent runs. func (s *Service) Status(ctx context.Context) (Status, error) { st, err := s.settings(ctx) if err != nil { return Status{}, err } t, err := s.times(ctx) if err != nil { return Status{}, err } rows, err := dbq.New(s.pool).ListBackupRuns(ctx, recentRuns) if err != nil { return Status{}, fmt.Errorf("backup: list runs: %w", err) } now := s.now() probs := problems(st) out := Status{ Config: view(st), State: verdict(st, probs, t, now), Problems: probs, LastSuccess: ptr(t.lastSuccess), FailingSince: ptr(t.failingSince), RunLimitSeconds: int(runLimit(st) / time.Second), Runs: make([]Run, 0, len(rows)), } if out.Problems == nil { out.Problems = []string{} } if st.IncludeMusic { if out.Music, err = s.musicProgress(ctx); err != nil { return Status{}, err } } s.mu.Lock() out.RunningSince = ptr(s.runningSince) s.mu.Unlock() if st.Enabled && len(probs) == 0 { next := due(now, int(st.RunHour), t) if next.Before(now) { // Due now: the next tick starts it. next = now } out.NextRun = &next } for _, r := range rows { out.Runs = append(out.Runs, Run{ ID: uuidText(r.ID), Trigger: r.Trigger, Status: r.Status, StartedAt: r.StartedAt.Time, FinishedAt: ptr(timeOf(r.FinishedAt)), Location: r.Location, Bytes: r.Bytes, Error: r.Error, MusicFiles: int(r.MusicFiles), MusicBytes: r.MusicBytes, MusicNote: r.MusicNote, }) } return out, nil } func bucketOf(st dbq.BackupSetting) (Bucket, error) { key, err := secret.Open(secret.BackupSecretKey.Purpose(), "", st.SecretKey) if err != nil { return Bucket{}, fmt.Errorf("backup: the stored secret key doesn't open; enter it again: %w", err) } return Bucket{Endpoint: st.Endpoint, Name: st.Bucket, AccessKey: st.AccessKey, SecretKey: key}, nil } // Test checks the destination u describes, falling back to the stored // secret key when u leaves it empty. It only reads: it lists the backups // already there and writes nothing. func (s *Service) Test(ctx context.Context, u Update) (TestResult, error) { st, err := s.settings(ctx) if err != nil { return TestResult{}, err } st, err = merge(st, u) if err != nil { return TestResult{}, err } if probs := problems(st); len(probs) > 0 { return TestResult{}, fmt.Errorf("%w: %s", ErrNotReady, strings.Join(probs, " ")) } ctx, cancel := context.WithTimeout(ctx, testTimeout) defer cancel() var res TestResult note := func(name string, at time.Time) { if !IsFileName(name) { return } res.Backups++ if res.Latest == nil || at.After(*res.Latest) { res.Latest = &at } } if st.Destination == DestinationFolder { if err := isFolder(st.Folder); err != nil { return res, err } entries, err := os.ReadDir(filepath.Join(st.Folder, databaseDir)) if err != nil && !errors.Is(err, os.ErrNotExist) { return res, fmt.Errorf("backup: read %s: %w", filepath.Join(st.Folder, databaseDir), err) } for _, e := range entries { if info, err := e.Info(); err == nil && info.Mode().IsRegular() { note(e.Name(), info.ModTime()) } } return res, nil } b, err := bucketOf(st) if err != nil { return res, err } objs, err := b.List(ctx, databaseDir+"/", 1000) if err != nil { return res, err } for _, o := range objs { note(path.Base(o.Key), o.LastModified) } return res, nil } // isFolder refuses a folder that isn't there. A backup never creates the // destination folder itself: a missing one usually means a volume that // isn't mounted, and writing beside it would fill the server's own disk. func isFolder(dir string) error { info, err := os.Stat(dir) if err != nil { return fmt.Errorf("backup: the folder %s isn't there; is its volume mounted? %w", dir, err) } if !info.IsDir() { return fmt.Errorf("backup: %s is a file, not a folder", dir) } return nil } // Start begins a backup in the background and returns at once. It returns // ErrRunning while one runs, ErrNotReady when the destination isn't set up, // and ErrClosed once the server is shutting down. func (s *Service) Start(ctx context.Context, trigger string) error { st, err := s.settings(ctx) if err != nil { return err } if len(problems(st)) > 0 { return ErrNotReady } s.mu.Lock() defer s.mu.Unlock() if s.closed { return ErrClosed } if !s.runningSince.IsZero() { return ErrRunning } s.runningSince = s.now() s.wg.Add(1) go func() { defer s.wg.Done() defer func() { s.mu.Lock() s.runningSince = time.Time{} s.mu.Unlock() }() s.run(trigger, st) }() return nil } // Running reports when the running backup started, or the zero time. func (s *Service) Running() time.Time { s.mu.Lock() defer s.mu.Unlock() return s.runningSince } // runLimit is the longest a run may take: the database file, and the // music's nightly budget when the backup includes the music. func runLimit(st dbq.BackupSetting) time.Duration { if st.IncludeMusic { return RunLimit + MusicBudget } return RunLimit } func (s *Service) run(trigger string, st dbq.BackupSetting) { ctx, cancel := context.WithTimeout(s.base, RunLimit) defer cancel() q := dbq.New(s.pool) started, err := q.StartBackupRun(ctx, trigger) if err != nil { s.logger.Error("backup: record the run's start", "err", err) return } location, size, err := s.write(ctx, st, started.StartedAt.Time) // The music goes only after the database file has landed, which also // proves the destination takes writes. var music musicSent if err == nil && st.IncludeMusic { mctx, mcancel := context.WithTimeout(s.base, MusicBudget) music = s.sendMusic(mctx, st) mcancel() } // The finish is recorded on its own context, so a run stopped by a // shutdown or its time limit still says how it ended. fctx, fcancel := context.WithTimeout(context.Background(), 10*time.Second) defer fcancel() fin := dbq.FinishBackupRunParams{ ID: started.ID, Status: "succeeded", Location: location, Bytes: size, MusicFiles: int32(music.files), MusicBytes: music.bytes, MusicNote: music.note, } if err != nil { fin.Status, fin.Error = "failed", describe(err) s.logger.Warn("backup failed", "trigger", trigger, "err", err) } else { s.logger.Info("backup written", "trigger", trigger, "location", location, "bytes", size, "music_files", music.files, "music_bytes", music.bytes, "music_note", music.note) } if err := q.FinishBackupRun(fctx, fin); err != nil { s.logger.Error("backup: record the run's end", "err", err) } if _, err := q.PruneBackupRuns(fctx, pgtype.Timestamptz{Time: s.now().Add(-keepRunsFor), Valid: true}); err != nil { s.logger.Warn("backup: prune old runs", "err", err) } } // describe is a run's error as the panel shows it. func describe(err error) string { switch { case errors.Is(err, context.DeadlineExceeded): return fmt.Sprintf("The backup took longer than %d minutes and was stopped.", int(RunLimit/time.Minute)) case errors.Is(err, context.Canceled): return "The server stopped while the backup was running." } return strings.TrimPrefix(err.Error(), "backup: ") } // write snapshots the database into one backup file at the destination and // returns where it went and its size. func (s *Service) write(ctx context.Context, st dbq.BackupSetting, at time.Time) (string, int64, error) { if st.Destination == DestinationFolder { // Checked before the snapshot, so a missing volume costs nothing. if err := isFolder(st.Folder); err != nil { return "", 0, err } } c, err := Snapshot(ctx, s.pool) if err != nil { return "", 0, err } if st.Destination == DestinationFolder { dir := filepath.Join(st.Folder, databaseDir) if err := os.Mkdir(dir, 0o700); err != nil && !errors.Is(err, os.ErrExist) { return "", 0, fmt.Errorf("backup: create %s: %w", dir, err) } file, _, err := WriteFile(dir, c, at, s.build) if err != nil { return "", 0, err } info, err := os.Stat(file) if err != nil { return file, 0, err } return file, info.Size(), nil } b, err := bucketOf(st) if err != nil { return "", 0, err } // The file is written to a temporary file first: S3 signs over the // body's hash and needs its length before the upload starts. tmp, err := os.CreateTemp("", ".minstrel-backup-*.tar") if err != nil { return "", 0, fmt.Errorf("backup: create a temporary file: %w", err) } defer func() { _ = tmp.Close() _ = os.Remove(tmp.Name()) }() h := sha256.New() if _, err := Write(io.MultiWriter(tmp, h), c, at, s.build); err != nil { return "", 0, fmt.Errorf("backup: write the temporary file: %w", err) } size, err := tmp.Seek(0, io.SeekCurrent) if err == nil { _, err = tmp.Seek(0, io.SeekStart) } if err != nil { return "", 0, fmt.Errorf("backup: rewind the temporary file: %w", err) } key := databaseDir + "/" + FileName(at) body := Throttle(ctx, tmp, int64(st.UploadLimitKib)*1024) if err := b.Put(ctx, key, body, size, hex.EncodeToString(h.Sum(nil))); err != nil { return "", 0, err } return key, size, nil } // Tick starts a scheduled backup when one is due. A backup that is already // running, or settings that aren't ready, mean there's nothing to do. func (s *Service) Tick(ctx context.Context) { st, err := s.settings(ctx) if err != nil { s.logger.Warn("backup: tick", "err", err) return } if !st.Enabled || len(problems(st)) > 0 { return } t, err := s.times(ctx) if err != nil { s.logger.Warn("backup: tick", "err", err) return } if due(s.now(), int(st.RunHour), t).After(s.now()) { return } if err := s.Start(ctx, TriggerSchedule); err != nil && !errors.Is(err, ErrRunning) && !errors.Is(err, ErrClosed) { s.logger.Warn("backup: start the scheduled backup", "err", err) } } // Recover marks runs a stopped server left running as failed. It runs once // at startup, before the first tick. func (s *Service) Recover(ctx context.Context) { n, err := dbq.New(s.pool).FailUnfinishedBackupRuns(ctx) if err != nil { s.logger.Warn("backup: recover unfinished runs", "err", err) return } if n > 0 { s.logger.Warn("backup: a run was cut off by the last shutdown", "runs", n) } } // Loop recovers, then ticks until ctx ends. The first tick waits a full // interval, so a restart isn't followed straight away by a backup. func (s *Service) Loop(ctx context.Context) { s.Recover(ctx) t := time.NewTicker(tickEvery) defer t.Stop() for { select { case <-ctx.Done(): return case <-t.C: s.Tick(ctx) } } } // Close stops a running backup and waits for it to record how it ended. func (s *Service) Close() { s.mu.Lock() s.closed = true s.mu.Unlock() s.cancel() s.wg.Wait() }