feat: a backup is one uncompressed tar of per-table JSON Lines that stays the same bytes day to day (#6769)
The file is minstrel-<UTC stamp>.tar: manifest.json first, then tables/<table>.jsonl, rows one per line in primary-key order. The run's time lives only in the manifest and member headers are fixed, so a deduplicating off-site copy stores unchanged tables once. WriteFile writes to a hidden .part file, syncs it and renames it, so a watched folder never shows half a backup. Read refuses anything that isn't a whole backup: another format, a newer format version, a cut-off file, or a member holding fewer rows than the manifest says. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
@@ -0,0 +1,226 @@
|
|||||||
|
package backup
|
||||||
|
|
||||||
|
import (
|
||||||
|
"archive/tar"
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The backup file is laid out so that what didn't change between two days is
|
||||||
|
// the same bytes both times, which suits a deduplicating off-site copy such as
|
||||||
|
// Fabled Archives' restic:
|
||||||
|
//
|
||||||
|
// - Never compressed. Compression rewrites every byte after the first
|
||||||
|
// change, so no chunk would match again.
|
||||||
|
// - One member per table, tables/<table>.jsonl: one row per line, in
|
||||||
|
// primary-key order, keys sorted, no spacing.
|
||||||
|
// - What changes every run lives only in manifest.json, the first member.
|
||||||
|
// Member headers are fixed (mtime 0, owner 0, mode 0644).
|
||||||
|
// - A new dated name each run, so the target keeps a history and a sender
|
||||||
|
// only ever adds.
|
||||||
|
const (
|
||||||
|
FormatName = "minstrel-backup-tar"
|
||||||
|
FormatVersion = 1
|
||||||
|
|
||||||
|
manifestName = "manifest.json"
|
||||||
|
tablesDir = "tables/"
|
||||||
|
tableSuffix = ".jsonl"
|
||||||
|
namePrefix = "minstrel-"
|
||||||
|
nameSuffix = ".tar"
|
||||||
|
|
||||||
|
// maxMember bounds any one member a read takes in: far past the largest
|
||||||
|
// library's tracks table, small enough that a hostile file can't fill
|
||||||
|
// memory.
|
||||||
|
maxMember = 1 << 30
|
||||||
|
)
|
||||||
|
|
||||||
|
// Manifest is the file's first member.
|
||||||
|
type Manifest struct {
|
||||||
|
Format string `json:"format"`
|
||||||
|
FormatVersion int `json:"format_version"`
|
||||||
|
CreatedAt time.Time `json:"created_at"`
|
||||||
|
// Build is the Minstrel version that wrote the file.
|
||||||
|
Build string `json:"build"`
|
||||||
|
// Schema is the database's migration version when the file was written.
|
||||||
|
// A restore refuses a file from a newer schema than its own.
|
||||||
|
Schema int64 `json:"schema"`
|
||||||
|
// Rows is each table's row count, so a read can tell a whole member from
|
||||||
|
// a cut-off one.
|
||||||
|
Rows map[string]int `json:"rows"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// FileName is a backup's file name: minstrel-20261102T030000Z.tar, in UTC, so
|
||||||
|
// names sort by time.
|
||||||
|
func FileName(at time.Time) string {
|
||||||
|
return namePrefix + at.UTC().Format("20060102T150405Z") + nameSuffix
|
||||||
|
}
|
||||||
|
|
||||||
|
// IsFileName reports whether a name is one FileName gives.
|
||||||
|
func IsFileName(name string) bool {
|
||||||
|
stamp, ok := strings.CutPrefix(name, namePrefix)
|
||||||
|
if !ok {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
stamp, ok = strings.CutSuffix(stamp, nameSuffix)
|
||||||
|
if !ok {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
_, err := time.Parse("20060102T150405Z", stamp)
|
||||||
|
return err == nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Write writes c as a backup file to w and returns its manifest.
|
||||||
|
func Write(w io.Writer, c Contents, at time.Time, build string) (Manifest, error) {
|
||||||
|
names := make([]string, 0, len(c.Tables))
|
||||||
|
m := Manifest{
|
||||||
|
Format: FormatName,
|
||||||
|
FormatVersion: FormatVersion,
|
||||||
|
CreatedAt: at.UTC(),
|
||||||
|
Build: build,
|
||||||
|
Schema: c.Schema,
|
||||||
|
Rows: make(map[string]int, len(c.Tables)),
|
||||||
|
}
|
||||||
|
for name, lines := range c.Tables {
|
||||||
|
names = append(names, name)
|
||||||
|
m.Rows[name] = len(lines)
|
||||||
|
}
|
||||||
|
slices.Sort(names)
|
||||||
|
|
||||||
|
manifest, err := json.MarshalIndent(m, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
return m, err
|
||||||
|
}
|
||||||
|
tw := tar.NewWriter(w)
|
||||||
|
if err := member(tw, manifestName, append(manifest, '\n')); err != nil {
|
||||||
|
return m, err
|
||||||
|
}
|
||||||
|
for _, name := range names {
|
||||||
|
var body bytes.Buffer
|
||||||
|
for _, line := range c.Tables[name] {
|
||||||
|
body.Write(line)
|
||||||
|
body.WriteByte('\n')
|
||||||
|
}
|
||||||
|
if err := member(tw, tablesDir+name+tableSuffix, body.Bytes()); err != nil {
|
||||||
|
return m, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return m, tw.Close()
|
||||||
|
}
|
||||||
|
|
||||||
|
// WriteFile writes c into dir under its dated name. The bytes go to a hidden
|
||||||
|
// .part file first, are synced, and only then take the backup's name, so
|
||||||
|
// anything watching the folder never sees half a backup under a backup's
|
||||||
|
// name, and a run that fails leaves nothing behind.
|
||||||
|
func WriteFile(dir string, c Contents, at time.Time, build string) (string, Manifest, error) {
|
||||||
|
final := filepath.Join(dir, FileName(at))
|
||||||
|
part := filepath.Join(dir, "."+FileName(at)+".part")
|
||||||
|
f, err := os.OpenFile(part, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600)
|
||||||
|
if err != nil {
|
||||||
|
return "", Manifest{}, fmt.Errorf("backup: create %s: %w", part, err)
|
||||||
|
}
|
||||||
|
m, err := Write(f, c, at, build)
|
||||||
|
if err == nil {
|
||||||
|
err = f.Sync()
|
||||||
|
}
|
||||||
|
if cerr := f.Close(); err == nil {
|
||||||
|
err = cerr
|
||||||
|
}
|
||||||
|
if err == nil {
|
||||||
|
err = os.Rename(part, final)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
_ = os.Remove(part)
|
||||||
|
return "", m, fmt.Errorf("backup: write %s: %w", final, err)
|
||||||
|
}
|
||||||
|
if d, err := os.Open(dir); err == nil {
|
||||||
|
_ = d.Sync()
|
||||||
|
_ = d.Close()
|
||||||
|
}
|
||||||
|
return final, m, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func member(tw *tar.Writer, name string, data []byte) error {
|
||||||
|
hdr := &tar.Header{
|
||||||
|
Name: name,
|
||||||
|
Mode: 0o644,
|
||||||
|
Size: int64(len(data)),
|
||||||
|
ModTime: time.Unix(0, 0),
|
||||||
|
Typeflag: tar.TypeReg,
|
||||||
|
Format: tar.FormatUSTAR,
|
||||||
|
}
|
||||||
|
if err := tw.WriteHeader(hdr); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, err := tw.Write(data)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read reads a backup file, refusing anything that isn't a whole one.
|
||||||
|
func Read(r io.Reader) (Manifest, Contents, error) {
|
||||||
|
var m Manifest
|
||||||
|
c := Contents{Tables: map[string][][]byte{}}
|
||||||
|
tr := tar.NewReader(r)
|
||||||
|
first := true
|
||||||
|
for {
|
||||||
|
hdr, err := tr.Next()
|
||||||
|
if errors.Is(err, io.EOF) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return m, c, fmt.Errorf("%w: %v", ErrNotABackup, err)
|
||||||
|
}
|
||||||
|
if hdr.Size > maxMember {
|
||||||
|
return m, c, fmt.Errorf("%w: %s is too large", ErrNotABackup, hdr.Name)
|
||||||
|
}
|
||||||
|
data, err := io.ReadAll(io.LimitReader(tr, maxMember))
|
||||||
|
if err != nil {
|
||||||
|
return m, c, fmt.Errorf("%w: %v", ErrNotABackup, err)
|
||||||
|
}
|
||||||
|
if first {
|
||||||
|
first = false
|
||||||
|
if hdr.Name != manifestName || json.Unmarshal(data, &m) != nil || m.Format != FormatName {
|
||||||
|
return m, c, ErrNotABackup
|
||||||
|
}
|
||||||
|
if m.FormatVersion > FormatVersion {
|
||||||
|
return m, c, fmt.Errorf("%w: it was written by a newer Minstrel (format %d)", ErrNotABackup, m.FormatVersion)
|
||||||
|
}
|
||||||
|
c.Schema = m.Schema
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
table, ok := strings.CutPrefix(hdr.Name, tablesDir)
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if table, ok = strings.CutSuffix(table, tableSuffix); !ok || table == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var lines [][]byte
|
||||||
|
for line := range bytes.SplitSeq(data, []byte("\n")) {
|
||||||
|
if len(line) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !json.Valid(line) {
|
||||||
|
return m, c, fmt.Errorf("%w: a row in %s is not JSON", ErrNotABackup, table)
|
||||||
|
}
|
||||||
|
lines = append(lines, line)
|
||||||
|
}
|
||||||
|
c.Tables[table] = lines
|
||||||
|
}
|
||||||
|
if first {
|
||||||
|
return m, c, ErrNotABackup
|
||||||
|
}
|
||||||
|
for table, want := range m.Rows {
|
||||||
|
if got := len(c.Tables[table]); got != want {
|
||||||
|
return m, c, fmt.Errorf("%w: %s holds %d rows, the manifest says %d", ErrNotABackup, table, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return m, c, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,190 @@
|
|||||||
|
package backup
|
||||||
|
|
||||||
|
import (
|
||||||
|
"archive/tar"
|
||||||
|
"bytes"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func sample() Contents {
|
||||||
|
return Contents{
|
||||||
|
Schema: 98,
|
||||||
|
Tables: map[string][][]byte{
|
||||||
|
"users": {[]byte(`{"id":"u1","username":"ada"}`), []byte(`{"id":"u2","username":"bo"}`)},
|
||||||
|
"tracks": {[]byte(`{"file_path":"/music/a.flac","id":"t1"}`)},
|
||||||
|
"wants": {},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func members(t *testing.T, file []byte) map[string][]byte {
|
||||||
|
t.Helper()
|
||||||
|
out := map[string][]byte{}
|
||||||
|
tr := tar.NewReader(bytes.NewReader(file))
|
||||||
|
for {
|
||||||
|
hdr, err := tr.Next()
|
||||||
|
if errors.Is(err, io.EOF) {
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
data, _ := io.ReadAll(tr)
|
||||||
|
out[hdr.Name] = data
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestABackupFileReadsBackAsWritten(t *testing.T) {
|
||||||
|
var buf bytes.Buffer
|
||||||
|
at := time.Date(2026, 11, 2, 3, 0, 0, 0, time.UTC)
|
||||||
|
if _, err := Write(&buf, sample(), at, "2026.11.02.0300"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
m, c, err := Read(&buf)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if m.Schema != 98 || c.Schema != 98 || m.Build != "2026.11.02.0300" || !m.CreatedAt.Equal(at) {
|
||||||
|
t.Errorf("manifest = %+v", m)
|
||||||
|
}
|
||||||
|
want := sample()
|
||||||
|
for name, lines := range want.Tables {
|
||||||
|
got := c.Tables[name]
|
||||||
|
if len(got) != len(lines) {
|
||||||
|
t.Errorf("%s: %d rows, want %d", name, len(got), len(lines))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for i := range lines {
|
||||||
|
if !bytes.Equal(got[i], lines[i]) {
|
||||||
|
t.Errorf("%s row %d = %s, want %s", name, i, got[i], lines[i])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Two days over unchanged data differ only in the manifest, so a
|
||||||
|
// deduplicating copy stores the tables once.
|
||||||
|
func TestOnlyTheManifestChangesBetweenDays(t *testing.T) {
|
||||||
|
var day1, day2 bytes.Buffer
|
||||||
|
if _, err := Write(&day1, sample(), time.Date(2026, 11, 1, 3, 0, 0, 0, time.UTC), "a"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := Write(&day2, sample(), time.Date(2026, 11, 2, 3, 0, 0, 0, time.UTC), "b"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
a, b := members(t, day1.Bytes()), members(t, day2.Bytes())
|
||||||
|
if len(a) != 4 {
|
||||||
|
t.Fatalf("members = %d, want the manifest and three tables", len(a))
|
||||||
|
}
|
||||||
|
for name, data := range a {
|
||||||
|
same := bytes.Equal(data, b[name])
|
||||||
|
if name == manifestName && same {
|
||||||
|
t.Error("the manifest is identical across days; it should carry the run's time")
|
||||||
|
}
|
||||||
|
if name != manifestName && !same {
|
||||||
|
t.Errorf("%s differs between days over the same rows", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// The table members' bytes, headers included, sit unchanged in both files.
|
||||||
|
tracks := a[tablesDir+"tracks"+tableSuffix]
|
||||||
|
if !bytes.Contains(day1.Bytes(), tracks) || !bytes.Contains(day2.Bytes(), tracks) {
|
||||||
|
t.Error("the tracks member is not stored verbatim")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestReadRefusesWhatIsNotAWholeBackup(t *testing.T) {
|
||||||
|
var good bytes.Buffer
|
||||||
|
if _, err := Write(&good, sample(), time.Now(), "x"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
tarOf := func(name string, data []byte) []byte {
|
||||||
|
var buf bytes.Buffer
|
||||||
|
tw := tar.NewWriter(&buf)
|
||||||
|
_ = member(tw, name, data)
|
||||||
|
_ = tw.Close()
|
||||||
|
return buf.Bytes()
|
||||||
|
}
|
||||||
|
short := func() []byte {
|
||||||
|
// The manifest promises two users; the member holds one.
|
||||||
|
var buf bytes.Buffer
|
||||||
|
tw := tar.NewWriter(&buf)
|
||||||
|
_ = member(tw, manifestName, []byte(`{"format":"minstrel-backup-tar","format_version":1,"rows":{"users":2}}`))
|
||||||
|
_ = member(tw, tablesDir+"users"+tableSuffix, []byte(`{"id":"u1"}`+"\n"))
|
||||||
|
_ = tw.Close()
|
||||||
|
return buf.Bytes()
|
||||||
|
}
|
||||||
|
|
||||||
|
cases := map[string][]byte{
|
||||||
|
"empty": nil,
|
||||||
|
"not a tar": []byte("hello"),
|
||||||
|
"cut off": good.Bytes()[:good.Len()/2],
|
||||||
|
"no manifest": tarOf(tablesDir+"users"+tableSuffix, []byte(`{"id":"u1"}`)),
|
||||||
|
"another format": tarOf(manifestName, []byte(`{"format":"fablednotary-backup-tar","format_version":1}`)),
|
||||||
|
"a newer format": tarOf(manifestName, []byte(`{"format":"minstrel-backup-tar","format_version":99}`)),
|
||||||
|
"a member short": short(),
|
||||||
|
}
|
||||||
|
for name, file := range cases {
|
||||||
|
if _, _, err := Read(bytes.NewReader(file)); !errors.Is(err, ErrNotABackup) {
|
||||||
|
t.Errorf("%s: err = %v, want ErrNotABackup", name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWriteFileLandsUnderTheDatedNameOnly(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
at := time.Date(2026, 11, 2, 3, 0, 0, 0, time.UTC)
|
||||||
|
path, _, err := WriteFile(dir, sample(), at, "x")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if filepath.Base(path) != "minstrel-20261102T030000Z.tar" {
|
||||||
|
t.Errorf("path = %s", path)
|
||||||
|
}
|
||||||
|
entries, _ := os.ReadDir(dir)
|
||||||
|
if len(entries) != 1 || entries[0].Name() != filepath.Base(path) {
|
||||||
|
t.Errorf("the folder holds %v, want only the backup", entries)
|
||||||
|
}
|
||||||
|
f, err := os.Open(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer f.Close()
|
||||||
|
if _, _, err := Read(f); err != nil {
|
||||||
|
t.Errorf("the written file doesn't read: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAFailedWriteFileLeavesNothingBehind(t *testing.T) {
|
||||||
|
dir := filepath.Join(t.TempDir(), "missing")
|
||||||
|
if _, _, err := WriteFile(dir, sample(), time.Now(), "x"); err == nil {
|
||||||
|
t.Fatal("writing into a missing folder succeeded")
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(dir); !os.IsNotExist(err) {
|
||||||
|
t.Errorf("the failed write created %s", dir)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFileNamesSortByTimeAndAreRecognised(t *testing.T) {
|
||||||
|
earlier := FileName(time.Date(2026, 1, 9, 23, 0, 0, 0, time.UTC))
|
||||||
|
later := FileName(time.Date(2026, 1, 10, 1, 0, 0, 0, time.FixedZone("x", 5*3600)))
|
||||||
|
if earlier >= later {
|
||||||
|
t.Errorf("%s does not sort before %s", earlier, later)
|
||||||
|
}
|
||||||
|
for name, want := range map[string]bool{
|
||||||
|
earlier: true,
|
||||||
|
"minstrel-20261102T030000Z.tar": true,
|
||||||
|
".minstrel-20261102T030000Z.tar.part": false,
|
||||||
|
"minstrel-yesterday.tar": false,
|
||||||
|
"fablednotary-20261102T030000Z.tar": false,
|
||||||
|
} {
|
||||||
|
if IsFileName(name) != want {
|
||||||
|
t.Errorf("IsFileName(%q) = %v, want %v", name, !want, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user