minstrel/internal/backup/archive.go

package backup

import (
	"archive/tar"
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"slices"
	"strings"
	"time"
)

// The backup file is laid out so that what didn't change between two days is
// the same bytes both times, which suits a deduplicating off-site copy such as
// Fabled Archives' restic:
//
//   - Never compressed. Compression rewrites every byte after the first
//     change, so no chunk would match again.
//   - One member per table, tables/<table>.jsonl: one row per line, in
//     primary-key order, keys sorted, no spacing.
//   - What changes every run lives only in manifest.json, the first member.
//     Member headers are fixed (mtime 0, owner 0, mode 0644).
//   - A new dated name each run, so the target keeps a history and a sender
//     only ever adds.
const (
	FormatName    = "minstrel-backup-tar"
	FormatVersion = 1

	manifestName = "manifest.json"
	tablesDir    = "tables/"
	tableSuffix  = ".jsonl"
	namePrefix   = "minstrel-"
	nameSuffix   = ".tar"

	// maxMember bounds any one member a read takes in: far past the largest
	// library's tracks table, small enough that a hostile file can't fill
	// memory.
	maxMember = 1 << 30
)

// Manifest is the file's first member.
type Manifest struct {
	Format        string    `json:"format"`
	FormatVersion int       `json:"format_version"`
	CreatedAt     time.Time `json:"created_at"`
	// Build is the Minstrel version that wrote the file.
	Build string `json:"build"`
	// Schema is the database's migration version when the file was written.
	// A restore refuses a file from a newer schema than its own.
	Schema int64 `json:"schema"`
	// Rows is each table's row count, so a read can tell a whole member from
	// a cut-off one.
	Rows map[string]int `json:"rows"`
}

// FileName is a backup's file name: minstrel-20261102T030000Z.tar, in UTC, so
// names sort by time.
func FileName(at time.Time) string {
	return namePrefix + at.UTC().Format("20060102T150405Z") + nameSuffix
}

// IsFileName reports whether a name is one FileName gives.
func IsFileName(name string) bool {
	stamp, ok := strings.CutPrefix(name, namePrefix)
	if !ok {
		return false
	}
	stamp, ok = strings.CutSuffix(stamp, nameSuffix)
	if !ok {
		return false
	}
	_, err := time.Parse("20060102T150405Z", stamp)
	return err == nil
}

// Write writes c as a backup file to w and returns its manifest.
func Write(w io.Writer, c Contents, at time.Time, build string) (Manifest, error) {
	names := make([]string, 0, len(c.Tables))
	m := Manifest{
		Format:        FormatName,
		FormatVersion: FormatVersion,
		CreatedAt:     at.UTC(),
		Build:         build,
		Schema:        c.Schema,
		Rows:          make(map[string]int, len(c.Tables)),
	}
	for name, lines := range c.Tables {
		names = append(names, name)
		m.Rows[name] = len(lines)
	}
	slices.Sort(names)

	manifest, err := json.MarshalIndent(m, "", "  ")
	if err != nil {
		return m, err
	}
	tw := tar.NewWriter(w)
	if err := member(tw, manifestName, append(manifest, '\n')); err != nil {
		return m, err
	}
	for _, name := range names {
		var body bytes.Buffer
		for _, line := range c.Tables[name] {
			body.Write(line)
			body.WriteByte('\n')
		}
		if err := member(tw, tablesDir+name+tableSuffix, body.Bytes()); err != nil {
			return m, err
		}
	}
	return m, tw.Close()
}

// WriteFile writes c into dir under its dated name. The bytes go to a hidden
// .part file first, are synced, and only then take the backup's name, so
// anything watching the folder never sees half a backup under a backup's
// name, and a run that fails leaves nothing behind.
func WriteFile(dir string, c Contents, at time.Time, build string) (string, Manifest, error) {
	final := filepath.Join(dir, FileName(at))
	part := filepath.Join(dir, "."+FileName(at)+".part")
	f, err := os.OpenFile(part, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600)
	if err != nil {
		return "", Manifest{}, fmt.Errorf("backup: create %s: %w", part, err)
	}
	m, err := Write(f, c, at, build)
	if err == nil {
		err = f.Sync()
	}
	if cerr := f.Close(); err == nil {
		err = cerr
	}
	if err == nil {
		err = os.Rename(part, final)
	}
	if err != nil {
		_ = os.Remove(part)
		return "", m, fmt.Errorf("backup: write %s: %w", final, err)
	}
	if d, err := os.Open(dir); err == nil {
		_ = d.Sync()
		_ = d.Close()
	}
	return final, m, nil
}

func member(tw *tar.Writer, name string, data []byte) error {
	hdr := &tar.Header{
		Name:     name,
		Mode:     0o644,
		Size:     int64(len(data)),
		ModTime:  time.Unix(0, 0),
		Typeflag: tar.TypeReg,
		Format:   tar.FormatUSTAR,
	}
	if err := tw.WriteHeader(hdr); err != nil {
		return err
	}
	_, err := tw.Write(data)
	return err
}

// Read reads a backup file, refusing anything that isn't a whole one.
func Read(r io.Reader) (Manifest, Contents, error) {
	var m Manifest
	c := Contents{Tables: map[string][][]byte{}}
	tr := tar.NewReader(r)
	first := true
	for {
		hdr, err := tr.Next()
		if errors.Is(err, io.EOF) {
			break
		}
		if err != nil {
			return m, c, fmt.Errorf("%w: %w", ErrNotABackup, err)
		}
		if hdr.Size > maxMember {
			return m, c, fmt.Errorf("%w: %s is too large", ErrNotABackup, hdr.Name)
		}
		data, err := io.ReadAll(io.LimitReader(tr, maxMember))
		if err != nil {
			return m, c, fmt.Errorf("%w: %w", ErrNotABackup, err)
		}
		if first {
			first = false
			if hdr.Name != manifestName || json.Unmarshal(data, &m) != nil || m.Format != FormatName {
				return m, c, ErrNotABackup
			}
			if m.FormatVersion > FormatVersion {
				return m, c, fmt.Errorf("%w: it was written by a newer Minstrel (format %d)", ErrNotABackup, m.FormatVersion)
			}
			c.Schema = m.Schema
			continue
		}
		table, ok := strings.CutPrefix(hdr.Name, tablesDir)
		if !ok {
			continue
		}
		if table, ok = strings.CutSuffix(table, tableSuffix); !ok || table == "" {
			continue
		}
		var lines [][]byte
		for line := range bytes.SplitSeq(data, []byte("\n")) {
			if len(line) == 0 {
				continue
			}
			if !json.Valid(line) {
				return m, c, fmt.Errorf("%w: a row in %s is not JSON", ErrNotABackup, table)
			}
			lines = append(lines, line)
		}
		c.Tables[table] = lines
	}
	if first {
		return m, c, ErrNotABackup
	}
	for table, want := range m.Rows {
		if got := len(c.Tables[table]); got != want {
			return m, c, fmt.Errorf("%w: %s holds %d rows, the manifest says %d", ErrNotABackup, table, got, want)
		}
	}
	return m, c, nil
}