diff --git a/internal/backup/archive.go b/internal/backup/archive.go new file mode 100644 index 00000000..895ff506 --- /dev/null +++ b/internal/backup/archive.go @@ -0,0 +1,226 @@ +package backup + +import ( + "archive/tar" + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "slices" + "strings" + "time" +) + +// The backup file is laid out so that what didn't change between two days is +// the same bytes both times, which suits a deduplicating off-site copy such as +// Fabled Archives' restic: +// +// - Never compressed. Compression rewrites every byte after the first +// change, so no chunk would match again. +// - One member per table, tables/.jsonl: one row per line, in +// primary-key order, keys sorted, no spacing. +// - What changes every run lives only in manifest.json, the first member. +// Member headers are fixed (mtime 0, owner 0, mode 0644). +// - A new dated name each run, so the target keeps a history and a sender +// only ever adds. +const ( + FormatName = "minstrel-backup-tar" + FormatVersion = 1 + + manifestName = "manifest.json" + tablesDir = "tables/" + tableSuffix = ".jsonl" + namePrefix = "minstrel-" + nameSuffix = ".tar" + + // maxMember bounds any one member a read takes in: far past the largest + // library's tracks table, small enough that a hostile file can't fill + // memory. + maxMember = 1 << 30 +) + +// Manifest is the file's first member. +type Manifest struct { + Format string `json:"format"` + FormatVersion int `json:"format_version"` + CreatedAt time.Time `json:"created_at"` + // Build is the Minstrel version that wrote the file. + Build string `json:"build"` + // Schema is the database's migration version when the file was written. + // A restore refuses a file from a newer schema than its own. + Schema int64 `json:"schema"` + // Rows is each table's row count, so a read can tell a whole member from + // a cut-off one. + Rows map[string]int `json:"rows"` +} + +// FileName is a backup's file name: minstrel-20261102T030000Z.tar, in UTC, so +// names sort by time. +func FileName(at time.Time) string { + return namePrefix + at.UTC().Format("20060102T150405Z") + nameSuffix +} + +// IsFileName reports whether a name is one FileName gives. +func IsFileName(name string) bool { + stamp, ok := strings.CutPrefix(name, namePrefix) + if !ok { + return false + } + stamp, ok = strings.CutSuffix(stamp, nameSuffix) + if !ok { + return false + } + _, err := time.Parse("20060102T150405Z", stamp) + return err == nil +} + +// Write writes c as a backup file to w and returns its manifest. +func Write(w io.Writer, c Contents, at time.Time, build string) (Manifest, error) { + names := make([]string, 0, len(c.Tables)) + m := Manifest{ + Format: FormatName, + FormatVersion: FormatVersion, + CreatedAt: at.UTC(), + Build: build, + Schema: c.Schema, + Rows: make(map[string]int, len(c.Tables)), + } + for name, lines := range c.Tables { + names = append(names, name) + m.Rows[name] = len(lines) + } + slices.Sort(names) + + manifest, err := json.MarshalIndent(m, "", " ") + if err != nil { + return m, err + } + tw := tar.NewWriter(w) + if err := member(tw, manifestName, append(manifest, '\n')); err != nil { + return m, err + } + for _, name := range names { + var body bytes.Buffer + for _, line := range c.Tables[name] { + body.Write(line) + body.WriteByte('\n') + } + if err := member(tw, tablesDir+name+tableSuffix, body.Bytes()); err != nil { + return m, err + } + } + return m, tw.Close() +} + +// WriteFile writes c into dir under its dated name. The bytes go to a hidden +// .part file first, are synced, and only then take the backup's name, so +// anything watching the folder never sees half a backup under a backup's +// name, and a run that fails leaves nothing behind. +func WriteFile(dir string, c Contents, at time.Time, build string) (string, Manifest, error) { + final := filepath.Join(dir, FileName(at)) + part := filepath.Join(dir, "."+FileName(at)+".part") + f, err := os.OpenFile(part, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600) + if err != nil { + return "", Manifest{}, fmt.Errorf("backup: create %s: %w", part, err) + } + m, err := Write(f, c, at, build) + if err == nil { + err = f.Sync() + } + if cerr := f.Close(); err == nil { + err = cerr + } + if err == nil { + err = os.Rename(part, final) + } + if err != nil { + _ = os.Remove(part) + return "", m, fmt.Errorf("backup: write %s: %w", final, err) + } + if d, err := os.Open(dir); err == nil { + _ = d.Sync() + _ = d.Close() + } + return final, m, nil +} + +func member(tw *tar.Writer, name string, data []byte) error { + hdr := &tar.Header{ + Name: name, + Mode: 0o644, + Size: int64(len(data)), + ModTime: time.Unix(0, 0), + Typeflag: tar.TypeReg, + Format: tar.FormatUSTAR, + } + if err := tw.WriteHeader(hdr); err != nil { + return err + } + _, err := tw.Write(data) + return err +} + +// Read reads a backup file, refusing anything that isn't a whole one. +func Read(r io.Reader) (Manifest, Contents, error) { + var m Manifest + c := Contents{Tables: map[string][][]byte{}} + tr := tar.NewReader(r) + first := true + for { + hdr, err := tr.Next() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return m, c, fmt.Errorf("%w: %v", ErrNotABackup, err) + } + if hdr.Size > maxMember { + return m, c, fmt.Errorf("%w: %s is too large", ErrNotABackup, hdr.Name) + } + data, err := io.ReadAll(io.LimitReader(tr, maxMember)) + if err != nil { + return m, c, fmt.Errorf("%w: %v", ErrNotABackup, err) + } + if first { + first = false + if hdr.Name != manifestName || json.Unmarshal(data, &m) != nil || m.Format != FormatName { + return m, c, ErrNotABackup + } + if m.FormatVersion > FormatVersion { + return m, c, fmt.Errorf("%w: it was written by a newer Minstrel (format %d)", ErrNotABackup, m.FormatVersion) + } + c.Schema = m.Schema + continue + } + table, ok := strings.CutPrefix(hdr.Name, tablesDir) + if !ok { + continue + } + if table, ok = strings.CutSuffix(table, tableSuffix); !ok || table == "" { + continue + } + var lines [][]byte + for line := range bytes.SplitSeq(data, []byte("\n")) { + if len(line) == 0 { + continue + } + if !json.Valid(line) { + return m, c, fmt.Errorf("%w: a row in %s is not JSON", ErrNotABackup, table) + } + lines = append(lines, line) + } + c.Tables[table] = lines + } + if first { + return m, c, ErrNotABackup + } + for table, want := range m.Rows { + if got := len(c.Tables[table]); got != want { + return m, c, fmt.Errorf("%w: %s holds %d rows, the manifest says %d", ErrNotABackup, table, got, want) + } + } + return m, c, nil +} diff --git a/internal/backup/archive_test.go b/internal/backup/archive_test.go new file mode 100644 index 00000000..08f39a74 --- /dev/null +++ b/internal/backup/archive_test.go @@ -0,0 +1,190 @@ +package backup + +import ( + "archive/tar" + "bytes" + "errors" + "io" + "os" + "path/filepath" + "testing" + "time" +) + +func sample() Contents { + return Contents{ + Schema: 98, + Tables: map[string][][]byte{ + "users": {[]byte(`{"id":"u1","username":"ada"}`), []byte(`{"id":"u2","username":"bo"}`)}, + "tracks": {[]byte(`{"file_path":"/music/a.flac","id":"t1"}`)}, + "wants": {}, + }, + } +} + +func members(t *testing.T, file []byte) map[string][]byte { + t.Helper() + out := map[string][]byte{} + tr := tar.NewReader(bytes.NewReader(file)) + for { + hdr, err := tr.Next() + if errors.Is(err, io.EOF) { + return out + } + if err != nil { + t.Fatal(err) + } + data, _ := io.ReadAll(tr) + out[hdr.Name] = data + } +} + +func TestABackupFileReadsBackAsWritten(t *testing.T) { + var buf bytes.Buffer + at := time.Date(2026, 11, 2, 3, 0, 0, 0, time.UTC) + if _, err := Write(&buf, sample(), at, "2026.11.02.0300"); err != nil { + t.Fatal(err) + } + m, c, err := Read(&buf) + if err != nil { + t.Fatal(err) + } + if m.Schema != 98 || c.Schema != 98 || m.Build != "2026.11.02.0300" || !m.CreatedAt.Equal(at) { + t.Errorf("manifest = %+v", m) + } + want := sample() + for name, lines := range want.Tables { + got := c.Tables[name] + if len(got) != len(lines) { + t.Errorf("%s: %d rows, want %d", name, len(got), len(lines)) + continue + } + for i := range lines { + if !bytes.Equal(got[i], lines[i]) { + t.Errorf("%s row %d = %s, want %s", name, i, got[i], lines[i]) + } + } + } +} + +// Two days over unchanged data differ only in the manifest, so a +// deduplicating copy stores the tables once. +func TestOnlyTheManifestChangesBetweenDays(t *testing.T) { + var day1, day2 bytes.Buffer + if _, err := Write(&day1, sample(), time.Date(2026, 11, 1, 3, 0, 0, 0, time.UTC), "a"); err != nil { + t.Fatal(err) + } + if _, err := Write(&day2, sample(), time.Date(2026, 11, 2, 3, 0, 0, 0, time.UTC), "b"); err != nil { + t.Fatal(err) + } + a, b := members(t, day1.Bytes()), members(t, day2.Bytes()) + if len(a) != 4 { + t.Fatalf("members = %d, want the manifest and three tables", len(a)) + } + for name, data := range a { + same := bytes.Equal(data, b[name]) + if name == manifestName && same { + t.Error("the manifest is identical across days; it should carry the run's time") + } + if name != manifestName && !same { + t.Errorf("%s differs between days over the same rows", name) + } + } + // The table members' bytes, headers included, sit unchanged in both files. + tracks := a[tablesDir+"tracks"+tableSuffix] + if !bytes.Contains(day1.Bytes(), tracks) || !bytes.Contains(day2.Bytes(), tracks) { + t.Error("the tracks member is not stored verbatim") + } +} + +func TestReadRefusesWhatIsNotAWholeBackup(t *testing.T) { + var good bytes.Buffer + if _, err := Write(&good, sample(), time.Now(), "x"); err != nil { + t.Fatal(err) + } + + tarOf := func(name string, data []byte) []byte { + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + _ = member(tw, name, data) + _ = tw.Close() + return buf.Bytes() + } + short := func() []byte { + // The manifest promises two users; the member holds one. + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + _ = member(tw, manifestName, []byte(`{"format":"minstrel-backup-tar","format_version":1,"rows":{"users":2}}`)) + _ = member(tw, tablesDir+"users"+tableSuffix, []byte(`{"id":"u1"}`+"\n")) + _ = tw.Close() + return buf.Bytes() + } + + cases := map[string][]byte{ + "empty": nil, + "not a tar": []byte("hello"), + "cut off": good.Bytes()[:good.Len()/2], + "no manifest": tarOf(tablesDir+"users"+tableSuffix, []byte(`{"id":"u1"}`)), + "another format": tarOf(manifestName, []byte(`{"format":"fablednotary-backup-tar","format_version":1}`)), + "a newer format": tarOf(manifestName, []byte(`{"format":"minstrel-backup-tar","format_version":99}`)), + "a member short": short(), + } + for name, file := range cases { + if _, _, err := Read(bytes.NewReader(file)); !errors.Is(err, ErrNotABackup) { + t.Errorf("%s: err = %v, want ErrNotABackup", name, err) + } + } +} + +func TestWriteFileLandsUnderTheDatedNameOnly(t *testing.T) { + dir := t.TempDir() + at := time.Date(2026, 11, 2, 3, 0, 0, 0, time.UTC) + path, _, err := WriteFile(dir, sample(), at, "x") + if err != nil { + t.Fatal(err) + } + if filepath.Base(path) != "minstrel-20261102T030000Z.tar" { + t.Errorf("path = %s", path) + } + entries, _ := os.ReadDir(dir) + if len(entries) != 1 || entries[0].Name() != filepath.Base(path) { + t.Errorf("the folder holds %v, want only the backup", entries) + } + f, err := os.Open(path) + if err != nil { + t.Fatal(err) + } + defer f.Close() + if _, _, err := Read(f); err != nil { + t.Errorf("the written file doesn't read: %v", err) + } +} + +func TestAFailedWriteFileLeavesNothingBehind(t *testing.T) { + dir := filepath.Join(t.TempDir(), "missing") + if _, _, err := WriteFile(dir, sample(), time.Now(), "x"); err == nil { + t.Fatal("writing into a missing folder succeeded") + } + if _, err := os.Stat(dir); !os.IsNotExist(err) { + t.Errorf("the failed write created %s", dir) + } +} + +func TestFileNamesSortByTimeAndAreRecognised(t *testing.T) { + earlier := FileName(time.Date(2026, 1, 9, 23, 0, 0, 0, time.UTC)) + later := FileName(time.Date(2026, 1, 10, 1, 0, 0, 0, time.FixedZone("x", 5*3600))) + if earlier >= later { + t.Errorf("%s does not sort before %s", earlier, later) + } + for name, want := range map[string]bool{ + earlier: true, + "minstrel-20261102T030000Z.tar": true, + ".minstrel-20261102T030000Z.tar.part": false, + "minstrel-yesterday.tar": false, + "fablednotary-20261102T030000Z.tar": false, + } { + if IsFileName(name) != want { + t.Errorf("IsFileName(%q) = %v, want %v", name, !want, want) + } + } +}