feat: Minstrel backs itself up nightly to a folder or an S3 bucket, off until an admin switches it on (#6773)

Admin › Backups shows a verdict first (off, not set up, waiting, backed up, overdue, failing since when), Back up now, Test destination and the latest runs, with the settings beside them. - backup.Service: one run at a time, bounded at 30 minutes; the schedule ticks every 5 minutes, catches up a slot missed while the server was down, and retries a failure after an hour and then after as long as it has been failing. Runs are recorded in backup_runs, pruned after 90 days; a run cut off by a shutdown is marked failed at the next start. - A folder destination writes database/minstrel-<UTC stamp>.tar and refuses a folder that isn't there. An S3 destination stages the file, hashes it, and uploads it under database/ through a throttled reader (4 MiB/s by default, 0 for no limit). - Test destination checks the form as typed and only lists. - The bucket secret is sealed like every stored credential; backup_settings and backup_runs are left out of the backup itself. - The panel polls only while a run is going, capped at the run limit plus five minutes. The music switch named in the task is left to #6774, so no control ships that does nothing. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-02 14:50:55 -05:00
parent faf19ef67b
commit 9d324ab2d0
29 changed files with 2384 additions and 4 deletions
@@ -0,0 +1,711 @@
+package backup
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"log/slog"
+	"net/url"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/jackc/pgx/v5/pgtype"
+	"github.com/jackc/pgx/v5/pgxpool"
+
+	"git.fabledsword.com/bvandeusen/minstrel/internal/db/dbq"
+	"git.fabledsword.com/bvandeusen/minstrel/internal/secret"
+)
+
+// Where a backup can go.
+const (
+	DestinationFolder = "folder"
+	DestinationS3     = "s3"
+)
+
+// What started a run.
+const (
+	TriggerSchedule = "schedule"
+	TriggerManual   = "manual"
+)
+
+// The verdict a Status gives, worst first in the order an admin acts on them.
+const (
+	StateOff        = "off"        // the nightly backup is switched off
+	StateIncomplete = "incomplete" // switched on, but the destination isn't set up
+	StateFailing    = "failing"    // the latest run failed
+	StateWaiting    = "waiting"    // set up, and no backup has finished yet
+	StateOverdue    = "overdue"    // the latest good backup is more than a day old
+	StateOK         = "ok"
+)
+
+const (
+	// RunLimit bounds one run, snapshot and upload together. The web panel
+	// stops polling a run a little after this, since the run is over by then.
+	RunLimit = 30 * time.Minute
+	// A nightly backup that hasn't succeeded for this long is overdue; the
+	// slack past a day covers a run that starts late or takes a while.
+	overdueAfter = 26 * time.Hour
+	tickEvery    = 5 * time.Minute
+	// A failed run is tried again after at least this long, and after longer
+	// each time it fails again, so a destination that is down isn't hammered.
+	retryAfter = time.Hour
+	recentRuns = 10
+	// Run history older than this is pruned after each run.
+	keepRunsFor = 90 * 24 * time.Hour
+	testTimeout = 30 * time.Second
+	// The database backups' folder inside the destination. The music, when
+	// it is backed up, sits beside it in the same destination.
+	databaseDir = "database"
+)
+
+var (
+	// ErrRunning is returned by Start while a backup is running.
+	ErrRunning = errors.New("backup: a backup is already running")
+	// ErrNotReady is returned by Start when the destination isn't set up.
+	ErrNotReady = errors.New("backup: the destination isn't set up yet")
+	// ErrClosed is returned by Start once the server is shutting down.
+	ErrClosed = errors.New("backup: the server is shutting down")
+)
+
+// FieldError is a setting that can't be saved, with the reason in words.
+type FieldError struct {
+	Field   string `json:"field"`
+	Message string `json:"message"`
+}
+
+func (e *FieldError) Error() string { return e.Field + ": " + e.Message }
+
+// Config is the backup's settings as an admin sees them. The bucket's secret
+// key is never sent back, only whether one is stored.
+type Config struct {
+	Enabled        bool   `json:"enabled"`
+	Destination    string `json:"destination"`
+	Folder         string `json:"folder"`
+	Endpoint       string `json:"endpoint"`
+	Bucket         string `json:"bucket"`
+	AccessKey      string `json:"access_key"`
+	HasSecretKey   bool   `json:"has_secret_key"`
+	RunHour        int    `json:"run_hour"`
+	UploadLimitKiB int    `json:"upload_limit_kib"`
+}
+
+// Update is a save from the admin card. An empty SecretKey keeps the stored
+// one, so the key doesn't have to be typed again on every save.
+type Update struct {
+	Enabled        bool   `json:"enabled"`
+	Destination    string `json:"destination"`
+	Folder         string `json:"folder"`
+	Endpoint       string `json:"endpoint"`
+	Bucket         string `json:"bucket"`
+	AccessKey      string `json:"access_key"`
+	SecretKey      string `json:"secret_key"`
+	RunHour        int    `json:"run_hour"`
+	UploadLimitKiB int    `json:"upload_limit_kib"`
+}
+
+// Run is one attempt at a backup.
+type Run struct {
+	ID         string     `json:"id"`
+	Trigger    string     `json:"trigger"`
+	Status     string     `json:"status"`
+	StartedAt  time.Time  `json:"started_at"`
+	FinishedAt *time.Time `json:"finished_at"`
+	Location   string     `json:"location"`
+	Bytes      int64      `json:"bytes"`
+	Error      string     `json:"error"`
+}
+
+// Status is everything the admin panel shows: a verdict, what stands in its
+// way, and the recent runs.
+type Status struct {
+	Config       Config     `json:"config"`
+	State        string     `json:"state"`
+	Problems     []string   `json:"problems"`
+	RunningSince *time.Time `json:"running_since"`
+	LastSuccess  *time.Time `json:"last_success"`
+	FailingSince *time.Time `json:"failing_since"`
+	// NextRun is when the schedule starts the next backup, or nil when it
+	// won't: switched off, or not set up.
+	NextRun         *time.Time `json:"next_run"`
+	RunLimitSeconds int        `json:"run_limit_seconds"`
+	Runs            []Run      `json:"runs"`
+}
+
+// TestResult is what a destination test found: how many backups are there
+// already and when the newest was made.
+type TestResult struct {
+	Backups int        `json:"backups"`
+	Latest  *time.Time `json:"latest"`
+}
+
+// Service runs Minstrel's own backup: nightly when switched on, or when an
+// admin asks. One run at a time; each is recorded in backup_runs.
+type Service struct {
+	pool   *pgxpool.Pool
+	logger *slog.Logger
+	build  string
+	now    func() time.Time
+
+	base   context.Context
+	cancel context.CancelFunc
+	wg     sync.WaitGroup
+
+	mu           sync.Mutex
+	runningSince time.Time
+	closed       bool
+}
+
+// NewService returns a Service over pool. build is the server's version,
+// written into each backup's manifest.
+func NewService(pool *pgxpool.Pool, logger *slog.Logger, build string) *Service {
+	base, cancel := context.WithCancel(context.Background())
+	return &Service{pool: pool, logger: logger, build: build, now: time.Now, base: base, cancel: cancel}
+}
+
+func (s *Service) settings(ctx context.Context) (dbq.BackupSetting, error) {
+	st, err := dbq.New(s.pool).GetBackupSettings(ctx)
+	if err != nil {
+		return st, fmt.Errorf("backup: read settings: %w", err)
+	}
+	return st, nil
+}
+
+func view(st dbq.BackupSetting) Config {
+	return Config{
+		Enabled:        st.Enabled,
+		Destination:    st.Destination,
+		Folder:         st.Folder,
+		Endpoint:       st.Endpoint,
+		Bucket:         st.Bucket,
+		AccessKey:      st.AccessKey,
+		HasSecretKey:   st.SecretKey != "",
+		RunHour:        int(st.RunHour),
+		UploadLimitKiB: int(st.UploadLimitKib),
+	}
+}
+
+// Config returns the settings as an admin sees them.
+func (s *Service) Config(ctx context.Context) (Config, error) {
+	st, err := s.settings(ctx)
+	return view(st), err
+}
+
+// merge applies u over the stored settings: it checks each field, and seals
+// a new secret key. It returns a *FieldError for a value that can't be kept.
+func merge(st dbq.BackupSetting, u Update) (dbq.BackupSetting, error) {
+	if u.Destination != DestinationFolder && u.Destination != DestinationS3 {
+		return st, &FieldError{"destination", "Choose a folder or an S3 bucket."}
+	}
+	if u.RunHour < 0 || u.RunHour > 23 {
+		return st, &FieldError{"run_hour", "The hour is from 0 to 23."}
+	}
+	if u.UploadLimitKiB < 0 || u.UploadLimitKiB > 1<<30 {
+		return st, &FieldError{"upload_limit_kib", "The upload limit is 0 (no limit) or more."}
+	}
+	folder := strings.TrimSpace(u.Folder)
+	if folder != "" && !filepath.IsAbs(folder) {
+		return st, &FieldError{"folder", "Give the folder's full path, starting with /."}
+	}
+	endpoint := strings.TrimRight(strings.TrimSpace(u.Endpoint), "/")
+	if endpoint != "" {
+		e, err := url.Parse(endpoint)
+		if err != nil || (e.Scheme != "http" && e.Scheme != "https") || e.Host == "" {
+			return st, &FieldError{"endpoint", "Give the endpoint as a URL, such as https://s3.example.com."}
+		}
+	}
+	st.Enabled = u.Enabled
+	st.Destination = u.Destination
+	st.Folder = filepath.Clean(folder)
+	if folder == "" {
+		st.Folder = ""
+	}
+	st.Endpoint = endpoint
+	st.Bucket = strings.TrimSpace(u.Bucket)
+	st.AccessKey = strings.TrimSpace(u.AccessKey)
+	st.RunHour = int16(u.RunHour)
+	st.UploadLimitKib = int32(u.UploadLimitKiB)
+	if u.SecretKey != "" {
+		sealed, err := secret.Seal(secret.BackupSecretKey.Purpose(), "", u.SecretKey)
+		if err != nil {
+			return st, err
+		}
+		st.SecretKey = sealed
+	}
+	return st, nil
+}
+
+// Save stores u. A value that can't be kept is a *FieldError and nothing is
+// stored. Settings that are incomplete still save; Status lists what's missing.
+func (s *Service) Save(ctx context.Context, u Update) error {
+	st, err := s.settings(ctx)
+	if err != nil {
+		return err
+	}
+	st, err = merge(st, u)
+	if err != nil {
+		return err
+	}
+	return dbq.New(s.pool).SaveBackupSettings(ctx, dbq.SaveBackupSettingsParams{
+		Enabled:        st.Enabled,
+		Destination:    st.Destination,
+		Folder:         st.Folder,
+		Endpoint:       st.Endpoint,
+		Bucket:         st.Bucket,
+		AccessKey:      st.AccessKey,
+		SecretKey:      st.SecretKey,
+		RunHour:        st.RunHour,
+		UploadLimitKib: st.UploadLimitKib,
+	})
+}
+
+// problems is what stops a backup from running with these settings, in words.
+func problems(st dbq.BackupSetting) []string {
+	var out []string
+	switch st.Destination {
+	case DestinationFolder:
+		if st.Folder == "" {
+			out = append(out, "Choose the folder backups are written to.")
+		}
+	case DestinationS3:
+		if st.Endpoint == "" {
+			out = append(out, "Enter the bucket's endpoint.")
+		}
+		if st.Bucket == "" {
+			out = append(out, "Enter the bucket's name.")
+		}
+		if st.AccessKey == "" {
+			out = append(out, "Enter the access key.")
+		}
+		if st.SecretKey == "" {
+			out = append(out, "Enter the secret key.")
+		}
+	default:
+		out = append(out, "Choose where backups go.")
+	}
+	return out
+}
+
+func timeOf(t pgtype.Timestamptz) time.Time {
+	if !t.Valid || t.Time.Unix() <= 0 {
+		return time.Time{}
+	}
+	return t.Time
+}
+
+func ptr(t time.Time) *time.Time {
+	if t.IsZero() {
+		return nil
+	}
+	return &t
+}
+
+// runTimes is what the verdict and the schedule turn on. A zero time is a run
+// that never happened.
+type runTimes struct {
+	lastSuccess, lastAttempt, failingSince time.Time
+}
+
+func (s *Service) times(ctx context.Context) (runTimes, error) {
+	row, err := dbq.New(s.pool).BackupRunTimes(ctx)
+	if err != nil {
+		return runTimes{}, fmt.Errorf("backup: read run times: %w", err)
+	}
+	return runTimes{timeOf(row.LastSuccess), timeOf(row.LastAttempt), timeOf(row.FailingSince)}, nil
+}
+
+// slotBefore is the latest scheduled start at or before now: today at hour
+// UTC, or yesterday's if that hour hasn't come yet.
+func slotBefore(now time.Time, hour int) time.Time {
+	now = now.UTC()
+	slot := time.Date(now.Year(), now.Month(), now.Day(), hour, 0, 0, 0, time.UTC)
+	if slot.After(now) {
+		slot = slot.AddDate(0, 0, -1)
+	}
+	return slot
+}
+
+// due reports when the schedule next starts a backup: now or earlier means
+// the next tick starts one. A slot that passed with no attempt since is due,
+// so a server that was down at the hour catches up. A run that failed is
+// tried again, waiting at least retryAfter and then as long again as it has
+// been failing, until it succeeds or the next slot comes.
+func due(now time.Time, hour int, t runTimes) time.Time {
+	slot := slotBefore(now, hour)
+	if t.lastAttempt.Before(slot) {
+		return slot
+	}
+	next := slot.AddDate(0, 0, 1)
+	if !t.failingSince.IsZero() {
+		wait := max(retryAfter, t.lastAttempt.Sub(t.failingSince))
+		if retry := t.lastAttempt.Add(wait); retry.Before(next) {
+			return retry
+		}
+	}
+	return next
+}
+
+func verdict(st dbq.BackupSetting, probs []string, t runTimes, now time.Time) string {
+	switch {
+	case !st.Enabled:
+		return StateOff
+	case len(probs) > 0:
+		return StateIncomplete
+	case !t.failingSince.IsZero():
+		return StateFailing
+	case t.lastSuccess.IsZero():
+		return StateWaiting
+	case now.Sub(t.lastSuccess) > overdueAfter:
+		return StateOverdue
+	}
+	return StateOK
+}
+
+func uuidText(u pgtype.UUID) string {
+	b := u.Bytes
+	return fmt.Sprintf("%x-%x-%x-%x-%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:16])
+}
+
+// Status is the verdict, what stands in its way, and the recent runs.
+func (s *Service) Status(ctx context.Context) (Status, error) {
+	st, err := s.settings(ctx)
+	if err != nil {
+		return Status{}, err
+	}
+	t, err := s.times(ctx)
+	if err != nil {
+		return Status{}, err
+	}
+	rows, err := dbq.New(s.pool).ListBackupRuns(ctx, recentRuns)
+	if err != nil {
+		return Status{}, fmt.Errorf("backup: list runs: %w", err)
+	}
+	now := s.now()
+	probs := problems(st)
+	out := Status{
+		Config:          view(st),
+		State:           verdict(st, probs, t, now),
+		Problems:        probs,
+		LastSuccess:     ptr(t.lastSuccess),
+		FailingSince:    ptr(t.failingSince),
+		RunLimitSeconds: int(RunLimit / time.Second),
+		Runs:            make([]Run, 0, len(rows)),
+	}
+	if out.Problems == nil {
+		out.Problems = []string{}
+	}
+	s.mu.Lock()
+	out.RunningSince = ptr(s.runningSince)
+	s.mu.Unlock()
+	if st.Enabled && len(probs) == 0 {
+		next := due(now, int(st.RunHour), t)
+		if next.Before(now) {
+			// Due now: the next tick starts it.
+			next = now
+		}
+		out.NextRun = &next
+	}
+	for _, r := range rows {
+		out.Runs = append(out.Runs, Run{
+			ID:         uuidText(r.ID),
+			Trigger:    r.Trigger,
+			Status:     r.Status,
+			StartedAt:  r.StartedAt.Time,
+			FinishedAt: ptr(timeOf(r.FinishedAt)),
+			Location:   r.Location,
+			Bytes:      r.Bytes,
+			Error:      r.Error,
+		})
+	}
+	return out, nil
+}
+
+func bucketOf(st dbq.BackupSetting) (Bucket, error) {
+	key, err := secret.Open(secret.BackupSecretKey.Purpose(), "", st.SecretKey)
+	if err != nil {
+		return Bucket{}, fmt.Errorf("backup: the stored secret key doesn't open; enter it again: %w", err)
+	}
+	return Bucket{Endpoint: st.Endpoint, Name: st.Bucket, AccessKey: st.AccessKey, SecretKey: key}, nil
+}
+
+// Test checks the destination u describes, falling back to the stored
+// secret key when u leaves it empty. It only reads: it lists the backups
+// already there and writes nothing.
+func (s *Service) Test(ctx context.Context, u Update) (TestResult, error) {
+	st, err := s.settings(ctx)
+	if err != nil {
+		return TestResult{}, err
+	}
+	st, err = merge(st, u)
+	if err != nil {
+		return TestResult{}, err
+	}
+	if probs := problems(st); len(probs) > 0 {
+		return TestResult{}, fmt.Errorf("%w: %s", ErrNotReady, strings.Join(probs, " "))
+	}
+	ctx, cancel := context.WithTimeout(ctx, testTimeout)
+	defer cancel()
+
+	var res TestResult
+	note := func(name string, at time.Time) {
+		if !IsFileName(name) {
+			return
+		}
+		res.Backups++
+		if res.Latest == nil || at.After(*res.Latest) {
+			res.Latest = &at
+		}
+	}
+	if st.Destination == DestinationFolder {
+		if err := isFolder(st.Folder); err != nil {
+			return res, err
+		}
+		entries, err := os.ReadDir(filepath.Join(st.Folder, databaseDir))
+		if err != nil && !errors.Is(err, os.ErrNotExist) {
+			return res, fmt.Errorf("backup: read %s: %w", filepath.Join(st.Folder, databaseDir), err)
+		}
+		for _, e := range entries {
+			if info, err := e.Info(); err == nil && info.Mode().IsRegular() {
+				note(e.Name(), info.ModTime())
+			}
+		}
+		return res, nil
+	}
+	b, err := bucketOf(st)
+	if err != nil {
+		return res, err
+	}
+	objs, err := b.List(ctx, databaseDir+"/", 1000)
+	if err != nil {
+		return res, err
+	}
+	for _, o := range objs {
+		note(path.Base(o.Key), o.LastModified)
+	}
+	return res, nil
+}
+
+// isFolder refuses a folder that isn't there. A backup never creates the
+// destination folder itself: a missing one usually means a volume that
+// isn't mounted, and writing beside it would fill the server's own disk.
+func isFolder(dir string) error {
+	info, err := os.Stat(dir)
+	if err != nil {
+		return fmt.Errorf("backup: the folder %s isn't there; is its volume mounted? %w", dir, err)
+	}
+	if !info.IsDir() {
+		return fmt.Errorf("backup: %s is a file, not a folder", dir)
+	}
+	return nil
+}
+
+// Start begins a backup in the background and returns at once. It returns
+// ErrRunning while one runs, ErrNotReady when the destination isn't set up,
+// and ErrClosed once the server is shutting down.
+func (s *Service) Start(ctx context.Context, trigger string) error {
+	st, err := s.settings(ctx)
+	if err != nil {
+		return err
+	}
+	if len(problems(st)) > 0 {
+		return ErrNotReady
+	}
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if s.closed {
+		return ErrClosed
+	}
+	if !s.runningSince.IsZero() {
+		return ErrRunning
+	}
+	s.runningSince = s.now()
+	s.wg.Add(1)
+	go func() {
+		defer s.wg.Done()
+		defer func() {
+			s.mu.Lock()
+			s.runningSince = time.Time{}
+			s.mu.Unlock()
+		}()
+		s.run(trigger, st)
+	}()
+	return nil
+}
+
+// Running reports when the running backup started, or the zero time.
+func (s *Service) Running() time.Time {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return s.runningSince
+}
+
+func (s *Service) run(trigger string, st dbq.BackupSetting) {
+	ctx, cancel := context.WithTimeout(s.base, RunLimit)
+	defer cancel()
+	q := dbq.New(s.pool)
+	started, err := q.StartBackupRun(ctx, trigger)
+	if err != nil {
+		s.logger.Error("backup: record the run's start", "err", err)
+		return
+	}
+	location, size, err := s.write(ctx, st, started.StartedAt.Time)
+
+	// The finish is recorded on its own context, so a run stopped by a
+	// shutdown or its time limit still says how it ended.
+	fctx, fcancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer fcancel()
+	fin := dbq.FinishBackupRunParams{ID: started.ID, Status: "succeeded", Location: location, Bytes: size}
+	if err != nil {
+		fin.Status, fin.Error = "failed", describe(err)
+		s.logger.Warn("backup failed", "trigger", trigger, "err", err)
+	} else {
+		s.logger.Info("backup written", "trigger", trigger, "location", location, "bytes", size)
+	}
+	if err := q.FinishBackupRun(fctx, fin); err != nil {
+		s.logger.Error("backup: record the run's end", "err", err)
+	}
+	if _, err := q.PruneBackupRuns(fctx, pgtype.Timestamptz{Time: s.now().Add(-keepRunsFor), Valid: true}); err != nil {
+		s.logger.Warn("backup: prune old runs", "err", err)
+	}
+}
+
+// describe is a run's error as the panel shows it.
+func describe(err error) string {
+	switch {
+	case errors.Is(err, context.DeadlineExceeded):
+		return fmt.Sprintf("The backup took longer than %d minutes and was stopped.", int(RunLimit/time.Minute))
+	case errors.Is(err, context.Canceled):
+		return "The server stopped while the backup was running."
+	}
+	return strings.TrimPrefix(err.Error(), "backup: ")
+}
+
+// write snapshots the database into one backup file at the destination and
+// returns where it went and its size.
+func (s *Service) write(ctx context.Context, st dbq.BackupSetting, at time.Time) (string, int64, error) {
+	if st.Destination == DestinationFolder {
+		// Checked before the snapshot, so a missing volume costs nothing.
+		if err := isFolder(st.Folder); err != nil {
+			return "", 0, err
+		}
+	}
+	c, err := Snapshot(ctx, s.pool)
+	if err != nil {
+		return "", 0, err
+	}
+	if st.Destination == DestinationFolder {
+		dir := filepath.Join(st.Folder, databaseDir)
+		if err := os.Mkdir(dir, 0o700); err != nil && !errors.Is(err, os.ErrExist) {
+			return "", 0, fmt.Errorf("backup: create %s: %w", dir, err)
+		}
+		file, _, err := WriteFile(dir, c, at, s.build)
+		if err != nil {
+			return "", 0, err
+		}
+		info, err := os.Stat(file)
+		if err != nil {
+			return file, 0, err
+		}
+		return file, info.Size(), nil
+	}
+
+	b, err := bucketOf(st)
+	if err != nil {
+		return "", 0, err
+	}
+	// The file is written to a temporary file first: S3 signs over the
+	// body's hash and needs its length before the upload starts.
+	tmp, err := os.CreateTemp("", ".minstrel-backup-*.tar")
+	if err != nil {
+		return "", 0, fmt.Errorf("backup: create a temporary file: %w", err)
+	}
+	defer func() {
+		_ = tmp.Close()
+		_ = os.Remove(tmp.Name())
+	}()
+	h := sha256.New()
+	if _, err := Write(io.MultiWriter(tmp, h), c, at, s.build); err != nil {
+		return "", 0, fmt.Errorf("backup: write the temporary file: %w", err)
+	}
+	size, err := tmp.Seek(0, io.SeekCurrent)
+	if err == nil {
+		_, err = tmp.Seek(0, io.SeekStart)
+	}
+	if err != nil {
+		return "", 0, fmt.Errorf("backup: rewind the temporary file: %w", err)
+	}
+	key := databaseDir + "/" + FileName(at)
+	body := Throttle(ctx, tmp, int64(st.UploadLimitKib)*1024)
+	if err := b.Put(ctx, key, body, size, hex.EncodeToString(h.Sum(nil))); err != nil {
+		return "", 0, err
+	}
+	return key, size, nil
+}
+
+// Tick starts a scheduled backup when one is due. A backup that is already
+// running, or settings that aren't ready, mean there's nothing to do.
+func (s *Service) Tick(ctx context.Context) {
+	st, err := s.settings(ctx)
+	if err != nil {
+		s.logger.Warn("backup: tick", "err", err)
+		return
+	}
+	if !st.Enabled || len(problems(st)) > 0 {
+		return
+	}
+	t, err := s.times(ctx)
+	if err != nil {
+		s.logger.Warn("backup: tick", "err", err)
+		return
+	}
+	if due(s.now(), int(st.RunHour), t).After(s.now()) {
+		return
+	}
+	if err := s.Start(ctx, TriggerSchedule); err != nil && !errors.Is(err, ErrRunning) && !errors.Is(err, ErrClosed) {
+		s.logger.Warn("backup: start the scheduled backup", "err", err)
+	}
+}
+
+// Recover marks runs a stopped server left running as failed. It runs once
+// at startup, before the first tick.
+func (s *Service) Recover(ctx context.Context) {
+	n, err := dbq.New(s.pool).FailUnfinishedBackupRuns(ctx)
+	if err != nil {
+		s.logger.Warn("backup: recover unfinished runs", "err", err)
+		return
+	}
+	if n > 0 {
+		s.logger.Warn("backup: a run was cut off by the last shutdown", "runs", n)
+	}
+}
+
+// Loop recovers, then ticks until ctx ends. The first tick waits a full
+// interval, so a restart isn't followed straight away by a backup.
+func (s *Service) Loop(ctx context.Context) {
+	s.Recover(ctx)
+	t := time.NewTicker(tickEvery)
+	defer t.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-t.C:
+			s.Tick(ctx)
+		}
+	}
+}
+
+// Close stops a running backup and waits for it to record how it ended.
+func (s *Service) Close() {
+	s.mu.Lock()
+	s.closed = true
+	s.mu.Unlock()
+	s.cancel()
+	s.wg.Wait()
+}