minstrel/internal/backup/service.go

package backup

import (
	"context"
	"crypto/sha256"
	"encoding/hex"
	"errors"
	"fmt"
	"io"
	"log/slog"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/jackc/pgx/v5/pgtype"
	"github.com/jackc/pgx/v5/pgxpool"

	"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
	"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
)

// Where a backup can go.
const (
	DestinationFolder = "folder"
	DestinationS3     = "s3"
)

// What started a run.
const (
	TriggerSchedule = "schedule"
	TriggerManual   = "manual"
)

// The verdict a Status gives, worst first in the order an admin acts on them.
const (
	StateOff        = "off"        // the nightly backup is switched off
	StateIncomplete = "incomplete" // switched on, but the destination isn't set up
	StateFailing    = "failing"    // the latest run failed
	StateWaiting    = "waiting"    // set up, and no backup has finished yet
	StateOverdue    = "overdue"    // the latest good backup is more than a day old
	StateOK         = "ok"
)

const (
	// RunLimit bounds one run, snapshot and upload together. The web panel
	// stops polling a run a little after this, since the run is over by then.
	RunLimit = 30 * time.Minute
	// A nightly backup that hasn't succeeded for this long is overdue; the
	// slack past a day covers a run that starts late or takes a while.
	overdueAfter = 26 * time.Hour
	tickEvery    = 5 * time.Minute
	// A failed run is tried again after at least this long, and after longer
	// each time it fails again, so a destination that is down isn't hammered.
	retryAfter = time.Hour
	recentRuns = 10
	// Run history older than this is pruned after each run.
	keepRunsFor = 90 * 24 * time.Hour
	testTimeout = 30 * time.Second
	// The database backups' folder inside the destination. The music, when
	// it is backed up, sits beside it in the same destination.
	databaseDir = "database"
)

var (
	// ErrRunning is returned by Start while a backup is running.
	ErrRunning = errors.New("backup: a backup is already running")
	// ErrNotReady is returned by Start when the destination isn't set up.
	ErrNotReady = errors.New("backup: the destination isn't set up yet")
	// ErrClosed is returned by Start once the server is shutting down.
	ErrClosed = errors.New("backup: the server is shutting down")
)

// FieldError is a setting that can't be saved, with the reason in words.
type FieldError struct {
	Field   string `json:"field"`
	Message string `json:"message"`
}

func (e *FieldError) Error() string { return e.Field + ": " + e.Message }

// Config is the backup's settings as an admin sees them. The bucket's secret
// key is never sent back, only whether one is stored.
type Config struct {
	Enabled        bool   `json:"enabled"`
	Destination    string `json:"destination"`
	Folder         string `json:"folder"`
	Endpoint       string `json:"endpoint"`
	Bucket         string `json:"bucket"`
	AccessKey      string `json:"access_key"`
	HasSecretKey   bool   `json:"has_secret_key"`
	RunHour        int    `json:"run_hour"`
	UploadLimitKiB int    `json:"upload_limit_kib"`
}

// Update is a save from the admin card. An empty SecretKey keeps the stored
// one, so the key doesn't have to be typed again on every save.
type Update struct {
	Enabled        bool   `json:"enabled"`
	Destination    string `json:"destination"`
	Folder         string `json:"folder"`
	Endpoint       string `json:"endpoint"`
	Bucket         string `json:"bucket"`
	AccessKey      string `json:"access_key"`
	SecretKey      string `json:"secret_key"`
	RunHour        int    `json:"run_hour"`
	UploadLimitKiB int    `json:"upload_limit_kib"`
}

// Run is one attempt at a backup.
type Run struct {
	ID         string     `json:"id"`
	Trigger    string     `json:"trigger"`
	Status     string     `json:"status"`
	StartedAt  time.Time  `json:"started_at"`
	FinishedAt *time.Time `json:"finished_at"`
	Location   string     `json:"location"`
	Bytes      int64      `json:"bytes"`
	Error      string     `json:"error"`
}

// Status is everything the admin panel shows: a verdict, what stands in its
// way, and the recent runs.
type Status struct {
	Config       Config     `json:"config"`
	State        string     `json:"state"`
	Problems     []string   `json:"problems"`
	RunningSince *time.Time `json:"running_since"`
	LastSuccess  *time.Time `json:"last_success"`
	FailingSince *time.Time `json:"failing_since"`
	// NextRun is when the schedule starts the next backup, or nil when it
	// won't: switched off, or not set up.
	NextRun         *time.Time `json:"next_run"`
	RunLimitSeconds int        `json:"run_limit_seconds"`
	Runs            []Run      `json:"runs"`
}

// TestResult is what a destination test found: how many backups are there
// already and when the newest was made.
type TestResult struct {
	Backups int        `json:"backups"`
	Latest  *time.Time `json:"latest"`
}

// Service runs Minstrel's own backup: nightly when switched on, or when an
// admin asks. One run at a time; each is recorded in backup_runs.
type Service struct {
	pool   *pgxpool.Pool
	logger *slog.Logger
	build  string
	now    func() time.Time

	base   context.Context
	cancel context.CancelFunc
	wg     sync.WaitGroup

	mu           sync.Mutex
	runningSince time.Time
	closed       bool
}

// NewService returns a Service over pool. build is the server's version,
// written into each backup's manifest.
func NewService(pool *pgxpool.Pool, logger *slog.Logger, build string) *Service {
	base, cancel := context.WithCancel(context.Background())
	return &Service{pool: pool, logger: logger, build: build, now: time.Now, base: base, cancel: cancel}
}

func (s *Service) settings(ctx context.Context) (dbq.BackupSetting, error) {
	st, err := dbq.New(s.pool).GetBackupSettings(ctx)
	if err != nil {
		return st, fmt.Errorf("backup: read settings: %w", err)
	}
	return st, nil
}

func view(st dbq.BackupSetting) Config {
	return Config{
		Enabled:        st.Enabled,
		Destination:    st.Destination,
		Folder:         st.Folder,
		Endpoint:       st.Endpoint,
		Bucket:         st.Bucket,
		AccessKey:      st.AccessKey,
		HasSecretKey:   st.SecretKey != "",
		RunHour:        int(st.RunHour),
		UploadLimitKiB: int(st.UploadLimitKib),
	}
}

// Config returns the settings as an admin sees them.
func (s *Service) Config(ctx context.Context) (Config, error) {
	st, err := s.settings(ctx)
	return view(st), err
}

// merge applies u over the stored settings: it checks each field, and seals
// a new secret key. It returns a *FieldError for a value that can't be kept.
func merge(st dbq.BackupSetting, u Update) (dbq.BackupSetting, error) {
	if u.Destination != DestinationFolder && u.Destination != DestinationS3 {
		return st, &FieldError{"destination", "Choose a folder or an S3 bucket."}
	}
	if u.RunHour < 0 || u.RunHour > 23 {
		return st, &FieldError{"run_hour", "The hour is from 0 to 23."}
	}
	if u.UploadLimitKiB < 0 || u.UploadLimitKiB > 1<<30 {
		return st, &FieldError{"upload_limit_kib", "The upload limit is 0 (no limit) or more."}
	}
	folder := strings.TrimSpace(u.Folder)
	if folder != "" && !filepath.IsAbs(folder) {
		return st, &FieldError{"folder", "Give the folder's full path, starting with /."}
	}
	endpoint := strings.TrimRight(strings.TrimSpace(u.Endpoint), "/")
	if endpoint != "" {
		e, err := url.Parse(endpoint)
		if err != nil || (e.Scheme != "http" && e.Scheme != "https") || e.Host == "" {
			return st, &FieldError{"endpoint", "Give the endpoint as a URL, such as https://s3.example.com."}
		}
	}
	st.Enabled = u.Enabled
	st.Destination = u.Destination
	st.Folder = filepath.Clean(folder)
	if folder == "" {
		st.Folder = ""
	}
	st.Endpoint = endpoint
	st.Bucket = strings.TrimSpace(u.Bucket)
	st.AccessKey = strings.TrimSpace(u.AccessKey)
	st.RunHour = int16(u.RunHour)
	st.UploadLimitKib = int32(u.UploadLimitKiB)
	if u.SecretKey != "" {
		sealed, err := secret.Seal(secret.BackupSecretKey.Purpose(), "", u.SecretKey)
		if err != nil {
			return st, err
		}
		st.SecretKey = sealed
	}
	return st, nil
}

// Save stores u. A value that can't be kept is a *FieldError and nothing is
// stored. Settings that are incomplete still save; Status lists what's missing.
func (s *Service) Save(ctx context.Context, u Update) error {
	st, err := s.settings(ctx)
	if err != nil {
		return err
	}
	st, err = merge(st, u)
	if err != nil {
		return err
	}
	return dbq.New(s.pool).SaveBackupSettings(ctx, dbq.SaveBackupSettingsParams{
		Enabled:        st.Enabled,
		Destination:    st.Destination,
		Folder:         st.Folder,
		Endpoint:       st.Endpoint,
		Bucket:         st.Bucket,
		AccessKey:      st.AccessKey,
		SecretKey:      st.SecretKey,
		RunHour:        st.RunHour,
		UploadLimitKib: st.UploadLimitKib,
	})
}

// problems is what stops a backup from running with these settings, in words.
func problems(st dbq.BackupSetting) []string {
	var out []string
	switch st.Destination {
	case DestinationFolder:
		if st.Folder == "" {
			out = append(out, "Choose the folder backups are written to.")
		}
	case DestinationS3:
		if st.Endpoint == "" {
			out = append(out, "Enter the bucket's endpoint.")
		}
		if st.Bucket == "" {
			out = append(out, "Enter the bucket's name.")
		}
		if st.AccessKey == "" {
			out = append(out, "Enter the access key.")
		}
		if st.SecretKey == "" {
			out = append(out, "Enter the secret key.")
		}
	default:
		out = append(out, "Choose where backups go.")
	}
	return out
}

func timeOf(t pgtype.Timestamptz) time.Time {
	if !t.Valid || t.Time.Unix() <= 0 {
		return time.Time{}
	}
	return t.Time
}

func ptr(t time.Time) *time.Time {
	if t.IsZero() {
		return nil
	}
	return &t
}

// runTimes is what the verdict and the schedule turn on. A zero time is a run
// that never happened.
type runTimes struct {
	lastSuccess, lastAttempt, failingSince time.Time
}

func (s *Service) times(ctx context.Context) (runTimes, error) {
	row, err := dbq.New(s.pool).BackupRunTimes(ctx)
	if err != nil {
		return runTimes{}, fmt.Errorf("backup: read run times: %w", err)
	}
	return runTimes{timeOf(row.LastSuccess), timeOf(row.LastAttempt), timeOf(row.FailingSince)}, nil
}

// slotBefore is the latest scheduled start at or before now: today at hour
// UTC, or yesterday's if that hour hasn't come yet.
func slotBefore(now time.Time, hour int) time.Time {
	now = now.UTC()
	slot := time.Date(now.Year(), now.Month(), now.Day(), hour, 0, 0, 0, time.UTC)
	if slot.After(now) {
		slot = slot.AddDate(0, 0, -1)
	}
	return slot
}

// due reports when the schedule next starts a backup: now or earlier means
// the next tick starts one. A slot that passed with no attempt since is due,
// so a server that was down at the hour catches up. A run that failed is
// tried again, waiting at least retryAfter and then as long again as it has
// been failing, until it succeeds or the next slot comes.
func due(now time.Time, hour int, t runTimes) time.Time {
	slot := slotBefore(now, hour)
	if t.lastAttempt.Before(slot) {
		return slot
	}
	next := slot.AddDate(0, 0, 1)
	if !t.failingSince.IsZero() {
		wait := max(retryAfter, t.lastAttempt.Sub(t.failingSince))
		if retry := t.lastAttempt.Add(wait); retry.Before(next) {
			return retry
		}
	}
	return next
}

func verdict(st dbq.BackupSetting, probs []string, t runTimes, now time.Time) string {
	switch {
	case !st.Enabled:
		return StateOff
	case len(probs) > 0:
		return StateIncomplete
	case !t.failingSince.IsZero():
		return StateFailing
	case t.lastSuccess.IsZero():
		return StateWaiting
	case now.Sub(t.lastSuccess) > overdueAfter:
		return StateOverdue
	}
	return StateOK
}

func uuidText(u pgtype.UUID) string {
	b := u.Bytes
	return fmt.Sprintf("%x-%x-%x-%x-%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:16])
}

// Status is the verdict, what stands in its way, and the recent runs.
func (s *Service) Status(ctx context.Context) (Status, error) {
	st, err := s.settings(ctx)
	if err != nil {
		return Status{}, err
	}
	t, err := s.times(ctx)
	if err != nil {
		return Status{}, err
	}
	rows, err := dbq.New(s.pool).ListBackupRuns(ctx, recentRuns)
	if err != nil {
		return Status{}, fmt.Errorf("backup: list runs: %w", err)
	}
	now := s.now()
	probs := problems(st)
	out := Status{
		Config:          view(st),
		State:           verdict(st, probs, t, now),
		Problems:        probs,
		LastSuccess:     ptr(t.lastSuccess),
		FailingSince:    ptr(t.failingSince),
		RunLimitSeconds: int(RunLimit / time.Second),
		Runs:            make([]Run, 0, len(rows)),
	}
	if out.Problems == nil {
		out.Problems = []string{}
	}
	s.mu.Lock()
	out.RunningSince = ptr(s.runningSince)
	s.mu.Unlock()
	if st.Enabled && len(probs) == 0 {
		next := due(now, int(st.RunHour), t)
		if next.Before(now) {
			// Due now: the next tick starts it.
			next = now
		}
		out.NextRun = &next
	}
	for _, r := range rows {
		out.Runs = append(out.Runs, Run{
			ID:         uuidText(r.ID),
			Trigger:    r.Trigger,
			Status:     r.Status,
			StartedAt:  r.StartedAt.Time,
			FinishedAt: ptr(timeOf(r.FinishedAt)),
			Location:   r.Location,
			Bytes:      r.Bytes,
			Error:      r.Error,
		})
	}
	return out, nil
}

func bucketOf(st dbq.BackupSetting) (Bucket, error) {
	key, err := secret.Open(secret.BackupSecretKey.Purpose(), "", st.SecretKey)
	if err != nil {
		return Bucket{}, fmt.Errorf("backup: the stored secret key doesn't open; enter it again: %w", err)
	}
	return Bucket{Endpoint: st.Endpoint, Name: st.Bucket, AccessKey: st.AccessKey, SecretKey: key}, nil
}

// Test checks the destination u describes, falling back to the stored
// secret key when u leaves it empty. It only reads: it lists the backups
// already there and writes nothing.
func (s *Service) Test(ctx context.Context, u Update) (TestResult, error) {
	st, err := s.settings(ctx)
	if err != nil {
		return TestResult{}, err
	}
	st, err = merge(st, u)
	if err != nil {
		return TestResult{}, err
	}
	if probs := problems(st); len(probs) > 0 {
		return TestResult{}, fmt.Errorf("%w: %s", ErrNotReady, strings.Join(probs, " "))
	}
	ctx, cancel := context.WithTimeout(ctx, testTimeout)
	defer cancel()

	var res TestResult
	note := func(name string, at time.Time) {
		if !IsFileName(name) {
			return
		}
		res.Backups++
		if res.Latest == nil || at.After(*res.Latest) {
			res.Latest = &at
		}
	}
	if st.Destination == DestinationFolder {
		if err := isFolder(st.Folder); err != nil {
			return res, err
		}
		entries, err := os.ReadDir(filepath.Join(st.Folder, databaseDir))
		if err != nil && !errors.Is(err, os.ErrNotExist) {
			return res, fmt.Errorf("backup: read %s: %w", filepath.Join(st.Folder, databaseDir), err)
		}
		for _, e := range entries {
			if info, err := e.Info(); err == nil && info.Mode().IsRegular() {
				note(e.Name(), info.ModTime())
			}
		}
		return res, nil
	}
	b, err := bucketOf(st)
	if err != nil {
		return res, err
	}
	objs, err := b.List(ctx, databaseDir+"/", 1000)
	if err != nil {
		return res, err
	}
	for _, o := range objs {
		note(path.Base(o.Key), o.LastModified)
	}
	return res, nil
}

// isFolder refuses a folder that isn't there. A backup never creates the
// destination folder itself: a missing one usually means a volume that
// isn't mounted, and writing beside it would fill the server's own disk.
func isFolder(dir string) error {
	info, err := os.Stat(dir)
	if err != nil {
		return fmt.Errorf("backup: the folder %s isn't there; is its volume mounted? %w", dir, err)
	}
	if !info.IsDir() {
		return fmt.Errorf("backup: %s is a file, not a folder", dir)
	}
	return nil
}

// Start begins a backup in the background and returns at once. It returns
// ErrRunning while one runs, ErrNotReady when the destination isn't set up,
// and ErrClosed once the server is shutting down.
func (s *Service) Start(ctx context.Context, trigger string) error {
	st, err := s.settings(ctx)
	if err != nil {
		return err
	}
	if len(problems(st)) > 0 {
		return ErrNotReady
	}
	s.mu.Lock()
	defer s.mu.Unlock()
	if s.closed {
		return ErrClosed
	}
	if !s.runningSince.IsZero() {
		return ErrRunning
	}
	s.runningSince = s.now()
	s.wg.Add(1)
	go func() {
		defer s.wg.Done()
		defer func() {
			s.mu.Lock()
			s.runningSince = time.Time{}
			s.mu.Unlock()
		}()
		s.run(trigger, st)
	}()
	return nil
}

// Running reports when the running backup started, or the zero time.
func (s *Service) Running() time.Time {
	s.mu.Lock()
	defer s.mu.Unlock()
	return s.runningSince
}

func (s *Service) run(trigger string, st dbq.BackupSetting) {
	ctx, cancel := context.WithTimeout(s.base, RunLimit)
	defer cancel()
	q := dbq.New(s.pool)
	started, err := q.StartBackupRun(ctx, trigger)
	if err != nil {
		s.logger.Error("backup: record the run's start", "err", err)
		return
	}
	location, size, err := s.write(ctx, st, started.StartedAt.Time)

	// The finish is recorded on its own context, so a run stopped by a
	// shutdown or its time limit still says how it ended.
	fctx, fcancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer fcancel()
	fin := dbq.FinishBackupRunParams{ID: started.ID, Status: "succeeded", Location: location, Bytes: size}
	if err != nil {
		fin.Status, fin.Error = "failed", describe(err)
		s.logger.Warn("backup failed", "trigger", trigger, "err", err)
	} else {
		s.logger.Info("backup written", "trigger", trigger, "location", location, "bytes", size)
	}
	if err := q.FinishBackupRun(fctx, fin); err != nil {
		s.logger.Error("backup: record the run's end", "err", err)
	}
	if _, err := q.PruneBackupRuns(fctx, pgtype.Timestamptz{Time: s.now().Add(-keepRunsFor), Valid: true}); err != nil {
		s.logger.Warn("backup: prune old runs", "err", err)
	}
}

// describe is a run's error as the panel shows it.
func describe(err error) string {
	switch {
	case errors.Is(err, context.DeadlineExceeded):
		return fmt.Sprintf("The backup took longer than %d minutes and was stopped.", int(RunLimit/time.Minute))
	case errors.Is(err, context.Canceled):
		return "The server stopped while the backup was running."
	}
	return strings.TrimPrefix(err.Error(), "backup: ")
}

// write snapshots the database into one backup file at the destination and
// returns where it went and its size.
func (s *Service) write(ctx context.Context, st dbq.BackupSetting, at time.Time) (string, int64, error) {
	if st.Destination == DestinationFolder {
		// Checked before the snapshot, so a missing volume costs nothing.
		if err := isFolder(st.Folder); err != nil {
			return "", 0, err
		}
	}
	c, err := Snapshot(ctx, s.pool)
	if err != nil {
		return "", 0, err
	}
	if st.Destination == DestinationFolder {
		dir := filepath.Join(st.Folder, databaseDir)
		if err := os.Mkdir(dir, 0o700); err != nil && !errors.Is(err, os.ErrExist) {
			return "", 0, fmt.Errorf("backup: create %s: %w", dir, err)
		}
		file, _, err := WriteFile(dir, c, at, s.build)
		if err != nil {
			return "", 0, err
		}
		info, err := os.Stat(file)
		if err != nil {
			return file, 0, err
		}
		return file, info.Size(), nil
	}

	b, err := bucketOf(st)
	if err != nil {
		return "", 0, err
	}
	// The file is written to a temporary file first: S3 signs over the
	// body's hash and needs its length before the upload starts.
	tmp, err := os.CreateTemp("", ".minstrel-backup-*.tar")
	if err != nil {
		return "", 0, fmt.Errorf("backup: create a temporary file: %w", err)
	}
	defer func() {
		_ = tmp.Close()
		_ = os.Remove(tmp.Name())
	}()
	h := sha256.New()
	if _, err := Write(io.MultiWriter(tmp, h), c, at, s.build); err != nil {
		return "", 0, fmt.Errorf("backup: write the temporary file: %w", err)
	}
	size, err := tmp.Seek(0, io.SeekCurrent)
	if err == nil {
		_, err = tmp.Seek(0, io.SeekStart)
	}
	if err != nil {
		return "", 0, fmt.Errorf("backup: rewind the temporary file: %w", err)
	}
	key := databaseDir + "/" + FileName(at)
	body := Throttle(ctx, tmp, int64(st.UploadLimitKib)*1024)
	if err := b.Put(ctx, key, body, size, hex.EncodeToString(h.Sum(nil))); err != nil {
		return "", 0, err
	}
	return key, size, nil
}

// Tick starts a scheduled backup when one is due. A backup that is already
// running, or settings that aren't ready, mean there's nothing to do.
func (s *Service) Tick(ctx context.Context) {
	st, err := s.settings(ctx)
	if err != nil {
		s.logger.Warn("backup: tick", "err", err)
		return
	}
	if !st.Enabled || len(problems(st)) > 0 {
		return
	}
	t, err := s.times(ctx)
	if err != nil {
		s.logger.Warn("backup: tick", "err", err)
		return
	}
	if due(s.now(), int(st.RunHour), t).After(s.now()) {
		return
	}
	if err := s.Start(ctx, TriggerSchedule); err != nil && !errors.Is(err, ErrRunning) && !errors.Is(err, ErrClosed) {
		s.logger.Warn("backup: start the scheduled backup", "err", err)
	}
}

// Recover marks runs a stopped server left running as failed. It runs once
// at startup, before the first tick.
func (s *Service) Recover(ctx context.Context) {
	n, err := dbq.New(s.pool).FailUnfinishedBackupRuns(ctx)
	if err != nil {
		s.logger.Warn("backup: recover unfinished runs", "err", err)
		return
	}
	if n > 0 {
		s.logger.Warn("backup: a run was cut off by the last shutdown", "runs", n)
	}
}

// Loop recovers, then ticks until ctx ends. The first tick waits a full
// interval, so a restart isn't followed straight away by a backup.
func (s *Service) Loop(ctx context.Context) {
	s.Recover(ctx)
	t := time.NewTicker(tickEvery)
	defer t.Stop()
	for {
		select {
		case <-ctx.Done():
			return
		case <-t.C:
			s.Tick(ctx)
		}
	}
}

// Close stops a running backup and waits for it to record how it ended.
func (s *Service) Close() {
	s.mu.Lock()
	s.closed = true
	s.mu.Unlock()
	s.cancel()
	s.wg.Wait()
}