feat: a backup is one uncompressed tar of per-table JSON Lines that stays the same bytes day to day (#6769)
The file is minstrel-<UTC stamp>.tar: manifest.json first, then tables/<table>.jsonl, rows one per line in primary-key order. The run's time lives only in the manifest and member headers are fixed, so a deduplicating off-site copy stores unchanged tables once. WriteFile writes to a hidden .part file, syncs it and renames it, so a watched folder never shows half a backup. Read refuses anything that isn't a whole backup: another format, a newer format version, a cut-off file, or a member holding fewer rows than the manifest says. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
@@ -0,0 +1,190 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func sample() Contents {
|
||||
return Contents{
|
||||
Schema: 98,
|
||||
Tables: map[string][][]byte{
|
||||
"users": {[]byte(`{"id":"u1","username":"ada"}`), []byte(`{"id":"u2","username":"bo"}`)},
|
||||
"tracks": {[]byte(`{"file_path":"/music/a.flac","id":"t1"}`)},
|
||||
"wants": {},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func members(t *testing.T, file []byte) map[string][]byte {
|
||||
t.Helper()
|
||||
out := map[string][]byte{}
|
||||
tr := tar.NewReader(bytes.NewReader(file))
|
||||
for {
|
||||
hdr, err := tr.Next()
|
||||
if errors.Is(err, io.EOF) {
|
||||
return out
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data, _ := io.ReadAll(tr)
|
||||
out[hdr.Name] = data
|
||||
}
|
||||
}
|
||||
|
||||
func TestABackupFileReadsBackAsWritten(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
at := time.Date(2026, 11, 2, 3, 0, 0, 0, time.UTC)
|
||||
if _, err := Write(&buf, sample(), at, "2026.11.02.0300"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, c, err := Read(&buf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m.Schema != 98 || c.Schema != 98 || m.Build != "2026.11.02.0300" || !m.CreatedAt.Equal(at) {
|
||||
t.Errorf("manifest = %+v", m)
|
||||
}
|
||||
want := sample()
|
||||
for name, lines := range want.Tables {
|
||||
got := c.Tables[name]
|
||||
if len(got) != len(lines) {
|
||||
t.Errorf("%s: %d rows, want %d", name, len(got), len(lines))
|
||||
continue
|
||||
}
|
||||
for i := range lines {
|
||||
if !bytes.Equal(got[i], lines[i]) {
|
||||
t.Errorf("%s row %d = %s, want %s", name, i, got[i], lines[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Two days over unchanged data differ only in the manifest, so a
|
||||
// deduplicating copy stores the tables once.
|
||||
func TestOnlyTheManifestChangesBetweenDays(t *testing.T) {
|
||||
var day1, day2 bytes.Buffer
|
||||
if _, err := Write(&day1, sample(), time.Date(2026, 11, 1, 3, 0, 0, 0, time.UTC), "a"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := Write(&day2, sample(), time.Date(2026, 11, 2, 3, 0, 0, 0, time.UTC), "b"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a, b := members(t, day1.Bytes()), members(t, day2.Bytes())
|
||||
if len(a) != 4 {
|
||||
t.Fatalf("members = %d, want the manifest and three tables", len(a))
|
||||
}
|
||||
for name, data := range a {
|
||||
same := bytes.Equal(data, b[name])
|
||||
if name == manifestName && same {
|
||||
t.Error("the manifest is identical across days; it should carry the run's time")
|
||||
}
|
||||
if name != manifestName && !same {
|
||||
t.Errorf("%s differs between days over the same rows", name)
|
||||
}
|
||||
}
|
||||
// The table members' bytes, headers included, sit unchanged in both files.
|
||||
tracks := a[tablesDir+"tracks"+tableSuffix]
|
||||
if !bytes.Contains(day1.Bytes(), tracks) || !bytes.Contains(day2.Bytes(), tracks) {
|
||||
t.Error("the tracks member is not stored verbatim")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadRefusesWhatIsNotAWholeBackup(t *testing.T) {
|
||||
var good bytes.Buffer
|
||||
if _, err := Write(&good, sample(), time.Now(), "x"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
tarOf := func(name string, data []byte) []byte {
|
||||
var buf bytes.Buffer
|
||||
tw := tar.NewWriter(&buf)
|
||||
_ = member(tw, name, data)
|
||||
_ = tw.Close()
|
||||
return buf.Bytes()
|
||||
}
|
||||
short := func() []byte {
|
||||
// The manifest promises two users; the member holds one.
|
||||
var buf bytes.Buffer
|
||||
tw := tar.NewWriter(&buf)
|
||||
_ = member(tw, manifestName, []byte(`{"format":"minstrel-backup-tar","format_version":1,"rows":{"users":2}}`))
|
||||
_ = member(tw, tablesDir+"users"+tableSuffix, []byte(`{"id":"u1"}`+"\n"))
|
||||
_ = tw.Close()
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
cases := map[string][]byte{
|
||||
"empty": nil,
|
||||
"not a tar": []byte("hello"),
|
||||
"cut off": good.Bytes()[:good.Len()/2],
|
||||
"no manifest": tarOf(tablesDir+"users"+tableSuffix, []byte(`{"id":"u1"}`)),
|
||||
"another format": tarOf(manifestName, []byte(`{"format":"fablednotary-backup-tar","format_version":1}`)),
|
||||
"a newer format": tarOf(manifestName, []byte(`{"format":"minstrel-backup-tar","format_version":99}`)),
|
||||
"a member short": short(),
|
||||
}
|
||||
for name, file := range cases {
|
||||
if _, _, err := Read(bytes.NewReader(file)); !errors.Is(err, ErrNotABackup) {
|
||||
t.Errorf("%s: err = %v, want ErrNotABackup", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteFileLandsUnderTheDatedNameOnly(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
at := time.Date(2026, 11, 2, 3, 0, 0, 0, time.UTC)
|
||||
path, _, err := WriteFile(dir, sample(), at, "x")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if filepath.Base(path) != "minstrel-20261102T030000Z.tar" {
|
||||
t.Errorf("path = %s", path)
|
||||
}
|
||||
entries, _ := os.ReadDir(dir)
|
||||
if len(entries) != 1 || entries[0].Name() != filepath.Base(path) {
|
||||
t.Errorf("the folder holds %v, want only the backup", entries)
|
||||
}
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer f.Close()
|
||||
if _, _, err := Read(f); err != nil {
|
||||
t.Errorf("the written file doesn't read: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAFailedWriteFileLeavesNothingBehind(t *testing.T) {
|
||||
dir := filepath.Join(t.TempDir(), "missing")
|
||||
if _, _, err := WriteFile(dir, sample(), time.Now(), "x"); err == nil {
|
||||
t.Fatal("writing into a missing folder succeeded")
|
||||
}
|
||||
if _, err := os.Stat(dir); !os.IsNotExist(err) {
|
||||
t.Errorf("the failed write created %s", dir)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileNamesSortByTimeAndAreRecognised(t *testing.T) {
|
||||
earlier := FileName(time.Date(2026, 1, 9, 23, 0, 0, 0, time.UTC))
|
||||
later := FileName(time.Date(2026, 1, 10, 1, 0, 0, 0, time.FixedZone("x", 5*3600)))
|
||||
if earlier >= later {
|
||||
t.Errorf("%s does not sort before %s", earlier, later)
|
||||
}
|
||||
for name, want := range map[string]bool{
|
||||
earlier: true,
|
||||
"minstrel-20261102T030000Z.tar": true,
|
||||
".minstrel-20261102T030000Z.tar.part": false,
|
||||
"minstrel-yesterday.tar": false,
|
||||
"fablednotary-20261102T030000Z.tar": false,
|
||||
} {
|
||||
if IsFileName(name) != want {
|
||||
t.Errorf("IsFileName(%q) = %v, want %v", name, !want, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user