package backup import ( "archive/tar" "bytes" "encoding/json" "errors" "fmt" "io" "os" "path/filepath" "slices" "strings" "time" ) // The backup file is laid out so that what didn't change between two days is // the same bytes both times, which suits a deduplicating off-site copy such as // Fabled Archives' restic: // // - Never compressed. Compression rewrites every byte after the first // change, so no chunk would match again. // - One member per table, tables/.jsonl: one row per line, in // primary-key order, keys sorted, no spacing. // - What changes every run lives only in manifest.json, the first member. // Member headers are fixed (mtime 0, owner 0, mode 0644). // - A new dated name each run, so the target keeps a history and a sender // only ever adds. const ( FormatName = "minstrel-backup-tar" FormatVersion = 1 manifestName = "manifest.json" tablesDir = "tables/" tableSuffix = ".jsonl" namePrefix = "minstrel-" nameSuffix = ".tar" // maxMember bounds any one member a read takes in: far past the largest // library's tracks table, small enough that a hostile file can't fill // memory. maxMember = 1 << 30 ) // Manifest is the file's first member. type Manifest struct { Format string `json:"format"` FormatVersion int `json:"format_version"` CreatedAt time.Time `json:"created_at"` // Build is the Minstrel version that wrote the file. Build string `json:"build"` // Schema is the database's migration version when the file was written. // A restore refuses a file from a newer schema than its own. Schema int64 `json:"schema"` // Rows is each table's row count, so a read can tell a whole member from // a cut-off one. Rows map[string]int `json:"rows"` } // FileName is a backup's file name: minstrel-20261102T030000Z.tar, in UTC, so // names sort by time. func FileName(at time.Time) string { return namePrefix + at.UTC().Format("20060102T150405Z") + nameSuffix } // IsFileName reports whether a name is one FileName gives. func IsFileName(name string) bool { stamp, ok := strings.CutPrefix(name, namePrefix) if !ok { return false } stamp, ok = strings.CutSuffix(stamp, nameSuffix) if !ok { return false } _, err := time.Parse("20060102T150405Z", stamp) return err == nil } // Write writes c as a backup file to w and returns its manifest. func Write(w io.Writer, c Contents, at time.Time, build string) (Manifest, error) { names := make([]string, 0, len(c.Tables)) m := Manifest{ Format: FormatName, FormatVersion: FormatVersion, CreatedAt: at.UTC(), Build: build, Schema: c.Schema, Rows: make(map[string]int, len(c.Tables)), } for name, lines := range c.Tables { names = append(names, name) m.Rows[name] = len(lines) } slices.Sort(names) manifest, err := json.MarshalIndent(m, "", " ") if err != nil { return m, err } tw := tar.NewWriter(w) if err := member(tw, manifestName, append(manifest, '\n')); err != nil { return m, err } for _, name := range names { var body bytes.Buffer for _, line := range c.Tables[name] { body.Write(line) body.WriteByte('\n') } if err := member(tw, tablesDir+name+tableSuffix, body.Bytes()); err != nil { return m, err } } return m, tw.Close() } // WriteFile writes c into dir under its dated name. The bytes go to a hidden // .part file first, are synced, and only then take the backup's name, so // anything watching the folder never sees half a backup under a backup's // name, and a run that fails leaves nothing behind. func WriteFile(dir string, c Contents, at time.Time, build string) (string, Manifest, error) { final := filepath.Join(dir, FileName(at)) part := filepath.Join(dir, "."+FileName(at)+".part") f, err := os.OpenFile(part, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600) if err != nil { return "", Manifest{}, fmt.Errorf("backup: create %s: %w", part, err) } m, err := Write(f, c, at, build) if err == nil { err = f.Sync() } if cerr := f.Close(); err == nil { err = cerr } if err == nil { err = os.Rename(part, final) } if err != nil { _ = os.Remove(part) return "", m, fmt.Errorf("backup: write %s: %w", final, err) } if d, err := os.Open(dir); err == nil { _ = d.Sync() _ = d.Close() } return final, m, nil } func member(tw *tar.Writer, name string, data []byte) error { hdr := &tar.Header{ Name: name, Mode: 0o644, Size: int64(len(data)), ModTime: time.Unix(0, 0), Typeflag: tar.TypeReg, Format: tar.FormatUSTAR, } if err := tw.WriteHeader(hdr); err != nil { return err } _, err := tw.Write(data) return err } // Read reads a backup file, refusing anything that isn't a whole one. func Read(r io.Reader) (Manifest, Contents, error) { var m Manifest c := Contents{Tables: map[string][][]byte{}} tr := tar.NewReader(r) first := true for { hdr, err := tr.Next() if errors.Is(err, io.EOF) { break } if err != nil { return m, c, fmt.Errorf("%w: %w", ErrNotABackup, err) } if hdr.Size > maxMember { return m, c, fmt.Errorf("%w: %s is too large", ErrNotABackup, hdr.Name) } data, err := io.ReadAll(io.LimitReader(tr, maxMember)) if err != nil { return m, c, fmt.Errorf("%w: %w", ErrNotABackup, err) } if first { first = false if hdr.Name != manifestName || json.Unmarshal(data, &m) != nil || m.Format != FormatName { return m, c, ErrNotABackup } if m.FormatVersion > FormatVersion { return m, c, fmt.Errorf("%w: it was written by a newer Minstrel (format %d)", ErrNotABackup, m.FormatVersion) } c.Schema = m.Schema continue } table, ok := strings.CutPrefix(hdr.Name, tablesDir) if !ok { continue } if table, ok = strings.CutSuffix(table, tableSuffix); !ok || table == "" { continue } var lines [][]byte for line := range bytes.SplitSeq(data, []byte("\n")) { if len(line) == 0 { continue } if !json.Valid(line) { return m, c, fmt.Errorf("%w: a row in %s is not JSON", ErrNotABackup, table) } lines = append(lines, line) } c.Tables[table] = lines } if first { return m, c, ErrNotABackup } for table, want := range m.Rows { if got := len(c.Tables[table]); got != want { return m, c, fmt.Errorf("%w: %s holds %d rows, the manifest says %d", ErrNotABackup, table, got, want) } } return m, c, nil }