Compare commits
2
Commits
a67897a6cd
...
b6ae205862
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b6ae205862 | ||
|
|
8be69b752a |
@@ -0,0 +1,226 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The backup file is laid out so that what didn't change between two days is
|
||||
// the same bytes both times, which suits a deduplicating off-site copy such as
|
||||
// Fabled Archives' restic:
|
||||
//
|
||||
// - Never compressed. Compression rewrites every byte after the first
|
||||
// change, so no chunk would match again.
|
||||
// - One member per table, tables/<table>.jsonl: one row per line, in
|
||||
// primary-key order, keys sorted, no spacing.
|
||||
// - What changes every run lives only in manifest.json, the first member.
|
||||
// Member headers are fixed (mtime 0, owner 0, mode 0644).
|
||||
// - A new dated name each run, so the target keeps a history and a sender
|
||||
// only ever adds.
|
||||
const (
|
||||
FormatName = "minstrel-backup-tar"
|
||||
FormatVersion = 1
|
||||
|
||||
manifestName = "manifest.json"
|
||||
tablesDir = "tables/"
|
||||
tableSuffix = ".jsonl"
|
||||
namePrefix = "minstrel-"
|
||||
nameSuffix = ".tar"
|
||||
|
||||
// maxMember bounds any one member a read takes in: far past the largest
|
||||
// library's tracks table, small enough that a hostile file can't fill
|
||||
// memory.
|
||||
maxMember = 1 << 30
|
||||
)
|
||||
|
||||
// Manifest is the file's first member.
|
||||
type Manifest struct {
|
||||
Format string `json:"format"`
|
||||
FormatVersion int `json:"format_version"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
// Build is the Minstrel version that wrote the file.
|
||||
Build string `json:"build"`
|
||||
// Schema is the database's migration version when the file was written.
|
||||
// A restore refuses a file from a newer schema than its own.
|
||||
Schema int64 `json:"schema"`
|
||||
// Rows is each table's row count, so a read can tell a whole member from
|
||||
// a cut-off one.
|
||||
Rows map[string]int `json:"rows"`
|
||||
}
|
||||
|
||||
// FileName is a backup's file name: minstrel-20261102T030000Z.tar, in UTC, so
|
||||
// names sort by time.
|
||||
func FileName(at time.Time) string {
|
||||
return namePrefix + at.UTC().Format("20060102T150405Z") + nameSuffix
|
||||
}
|
||||
|
||||
// IsFileName reports whether a name is one FileName gives.
|
||||
func IsFileName(name string) bool {
|
||||
stamp, ok := strings.CutPrefix(name, namePrefix)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
stamp, ok = strings.CutSuffix(stamp, nameSuffix)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
_, err := time.Parse("20060102T150405Z", stamp)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// Write writes c as a backup file to w and returns its manifest.
|
||||
func Write(w io.Writer, c Contents, at time.Time, build string) (Manifest, error) {
|
||||
names := make([]string, 0, len(c.Tables))
|
||||
m := Manifest{
|
||||
Format: FormatName,
|
||||
FormatVersion: FormatVersion,
|
||||
CreatedAt: at.UTC(),
|
||||
Build: build,
|
||||
Schema: c.Schema,
|
||||
Rows: make(map[string]int, len(c.Tables)),
|
||||
}
|
||||
for name, lines := range c.Tables {
|
||||
names = append(names, name)
|
||||
m.Rows[name] = len(lines)
|
||||
}
|
||||
slices.Sort(names)
|
||||
|
||||
manifest, err := json.MarshalIndent(m, "", " ")
|
||||
if err != nil {
|
||||
return m, err
|
||||
}
|
||||
tw := tar.NewWriter(w)
|
||||
if err := member(tw, manifestName, append(manifest, '\n')); err != nil {
|
||||
return m, err
|
||||
}
|
||||
for _, name := range names {
|
||||
var body bytes.Buffer
|
||||
for _, line := range c.Tables[name] {
|
||||
body.Write(line)
|
||||
body.WriteByte('\n')
|
||||
}
|
||||
if err := member(tw, tablesDir+name+tableSuffix, body.Bytes()); err != nil {
|
||||
return m, err
|
||||
}
|
||||
}
|
||||
return m, tw.Close()
|
||||
}
|
||||
|
||||
// WriteFile writes c into dir under its dated name. The bytes go to a hidden
|
||||
// .part file first, are synced, and only then take the backup's name, so
|
||||
// anything watching the folder never sees half a backup under a backup's
|
||||
// name, and a run that fails leaves nothing behind.
|
||||
func WriteFile(dir string, c Contents, at time.Time, build string) (string, Manifest, error) {
|
||||
final := filepath.Join(dir, FileName(at))
|
||||
part := filepath.Join(dir, "."+FileName(at)+".part")
|
||||
f, err := os.OpenFile(part, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600)
|
||||
if err != nil {
|
||||
return "", Manifest{}, fmt.Errorf("backup: create %s: %w", part, err)
|
||||
}
|
||||
m, err := Write(f, c, at, build)
|
||||
if err == nil {
|
||||
err = f.Sync()
|
||||
}
|
||||
if cerr := f.Close(); err == nil {
|
||||
err = cerr
|
||||
}
|
||||
if err == nil {
|
||||
err = os.Rename(part, final)
|
||||
}
|
||||
if err != nil {
|
||||
_ = os.Remove(part)
|
||||
return "", m, fmt.Errorf("backup: write %s: %w", final, err)
|
||||
}
|
||||
if d, err := os.Open(dir); err == nil {
|
||||
_ = d.Sync()
|
||||
_ = d.Close()
|
||||
}
|
||||
return final, m, nil
|
||||
}
|
||||
|
||||
func member(tw *tar.Writer, name string, data []byte) error {
|
||||
hdr := &tar.Header{
|
||||
Name: name,
|
||||
Mode: 0o644,
|
||||
Size: int64(len(data)),
|
||||
ModTime: time.Unix(0, 0),
|
||||
Typeflag: tar.TypeReg,
|
||||
Format: tar.FormatUSTAR,
|
||||
}
|
||||
if err := tw.WriteHeader(hdr); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := tw.Write(data)
|
||||
return err
|
||||
}
|
||||
|
||||
// Read reads a backup file, refusing anything that isn't a whole one.
|
||||
func Read(r io.Reader) (Manifest, Contents, error) {
|
||||
var m Manifest
|
||||
c := Contents{Tables: map[string][][]byte{}}
|
||||
tr := tar.NewReader(r)
|
||||
first := true
|
||||
for {
|
||||
hdr, err := tr.Next()
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return m, c, fmt.Errorf("%w: %v", ErrNotABackup, err)
|
||||
}
|
||||
if hdr.Size > maxMember {
|
||||
return m, c, fmt.Errorf("%w: %s is too large", ErrNotABackup, hdr.Name)
|
||||
}
|
||||
data, err := io.ReadAll(io.LimitReader(tr, maxMember))
|
||||
if err != nil {
|
||||
return m, c, fmt.Errorf("%w: %v", ErrNotABackup, err)
|
||||
}
|
||||
if first {
|
||||
first = false
|
||||
if hdr.Name != manifestName || json.Unmarshal(data, &m) != nil || m.Format != FormatName {
|
||||
return m, c, ErrNotABackup
|
||||
}
|
||||
if m.FormatVersion > FormatVersion {
|
||||
return m, c, fmt.Errorf("%w: it was written by a newer Minstrel (format %d)", ErrNotABackup, m.FormatVersion)
|
||||
}
|
||||
c.Schema = m.Schema
|
||||
continue
|
||||
}
|
||||
table, ok := strings.CutPrefix(hdr.Name, tablesDir)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if table, ok = strings.CutSuffix(table, tableSuffix); !ok || table == "" {
|
||||
continue
|
||||
}
|
||||
var lines [][]byte
|
||||
for line := range bytes.SplitSeq(data, []byte("\n")) {
|
||||
if len(line) == 0 {
|
||||
continue
|
||||
}
|
||||
if !json.Valid(line) {
|
||||
return m, c, fmt.Errorf("%w: a row in %s is not JSON", ErrNotABackup, table)
|
||||
}
|
||||
lines = append(lines, line)
|
||||
}
|
||||
c.Tables[table] = lines
|
||||
}
|
||||
if first {
|
||||
return m, c, ErrNotABackup
|
||||
}
|
||||
for table, want := range m.Rows {
|
||||
if got := len(c.Tables[table]); got != want {
|
||||
return m, c, fmt.Errorf("%w: %s holds %d rows, the manifest says %d", ErrNotABackup, table, got, want)
|
||||
}
|
||||
}
|
||||
return m, c, nil
|
||||
}
|
||||
@@ -0,0 +1,190 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func sample() Contents {
|
||||
return Contents{
|
||||
Schema: 98,
|
||||
Tables: map[string][][]byte{
|
||||
"users": {[]byte(`{"id":"u1","username":"ada"}`), []byte(`{"id":"u2","username":"bo"}`)},
|
||||
"tracks": {[]byte(`{"file_path":"/music/a.flac","id":"t1"}`)},
|
||||
"wants": {},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func members(t *testing.T, file []byte) map[string][]byte {
|
||||
t.Helper()
|
||||
out := map[string][]byte{}
|
||||
tr := tar.NewReader(bytes.NewReader(file))
|
||||
for {
|
||||
hdr, err := tr.Next()
|
||||
if errors.Is(err, io.EOF) {
|
||||
return out
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data, _ := io.ReadAll(tr)
|
||||
out[hdr.Name] = data
|
||||
}
|
||||
}
|
||||
|
||||
func TestABackupFileReadsBackAsWritten(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
at := time.Date(2026, 11, 2, 3, 0, 0, 0, time.UTC)
|
||||
if _, err := Write(&buf, sample(), at, "2026.11.02.0300"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, c, err := Read(&buf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m.Schema != 98 || c.Schema != 98 || m.Build != "2026.11.02.0300" || !m.CreatedAt.Equal(at) {
|
||||
t.Errorf("manifest = %+v", m)
|
||||
}
|
||||
want := sample()
|
||||
for name, lines := range want.Tables {
|
||||
got := c.Tables[name]
|
||||
if len(got) != len(lines) {
|
||||
t.Errorf("%s: %d rows, want %d", name, len(got), len(lines))
|
||||
continue
|
||||
}
|
||||
for i := range lines {
|
||||
if !bytes.Equal(got[i], lines[i]) {
|
||||
t.Errorf("%s row %d = %s, want %s", name, i, got[i], lines[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Two days over unchanged data differ only in the manifest, so a
|
||||
// deduplicating copy stores the tables once.
|
||||
func TestOnlyTheManifestChangesBetweenDays(t *testing.T) {
|
||||
var day1, day2 bytes.Buffer
|
||||
if _, err := Write(&day1, sample(), time.Date(2026, 11, 1, 3, 0, 0, 0, time.UTC), "a"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := Write(&day2, sample(), time.Date(2026, 11, 2, 3, 0, 0, 0, time.UTC), "b"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a, b := members(t, day1.Bytes()), members(t, day2.Bytes())
|
||||
if len(a) != 4 {
|
||||
t.Fatalf("members = %d, want the manifest and three tables", len(a))
|
||||
}
|
||||
for name, data := range a {
|
||||
same := bytes.Equal(data, b[name])
|
||||
if name == manifestName && same {
|
||||
t.Error("the manifest is identical across days; it should carry the run's time")
|
||||
}
|
||||
if name != manifestName && !same {
|
||||
t.Errorf("%s differs between days over the same rows", name)
|
||||
}
|
||||
}
|
||||
// The table members' bytes, headers included, sit unchanged in both files.
|
||||
tracks := a[tablesDir+"tracks"+tableSuffix]
|
||||
if !bytes.Contains(day1.Bytes(), tracks) || !bytes.Contains(day2.Bytes(), tracks) {
|
||||
t.Error("the tracks member is not stored verbatim")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadRefusesWhatIsNotAWholeBackup(t *testing.T) {
|
||||
var good bytes.Buffer
|
||||
if _, err := Write(&good, sample(), time.Now(), "x"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
tarOf := func(name string, data []byte) []byte {
|
||||
var buf bytes.Buffer
|
||||
tw := tar.NewWriter(&buf)
|
||||
_ = member(tw, name, data)
|
||||
_ = tw.Close()
|
||||
return buf.Bytes()
|
||||
}
|
||||
short := func() []byte {
|
||||
// The manifest promises two users; the member holds one.
|
||||
var buf bytes.Buffer
|
||||
tw := tar.NewWriter(&buf)
|
||||
_ = member(tw, manifestName, []byte(`{"format":"minstrel-backup-tar","format_version":1,"rows":{"users":2}}`))
|
||||
_ = member(tw, tablesDir+"users"+tableSuffix, []byte(`{"id":"u1"}`+"\n"))
|
||||
_ = tw.Close()
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
cases := map[string][]byte{
|
||||
"empty": nil,
|
||||
"not a tar": []byte("hello"),
|
||||
"cut off": good.Bytes()[:good.Len()/2],
|
||||
"no manifest": tarOf(tablesDir+"users"+tableSuffix, []byte(`{"id":"u1"}`)),
|
||||
"another format": tarOf(manifestName, []byte(`{"format":"fablednotary-backup-tar","format_version":1}`)),
|
||||
"a newer format": tarOf(manifestName, []byte(`{"format":"minstrel-backup-tar","format_version":99}`)),
|
||||
"a member short": short(),
|
||||
}
|
||||
for name, file := range cases {
|
||||
if _, _, err := Read(bytes.NewReader(file)); !errors.Is(err, ErrNotABackup) {
|
||||
t.Errorf("%s: err = %v, want ErrNotABackup", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteFileLandsUnderTheDatedNameOnly(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
at := time.Date(2026, 11, 2, 3, 0, 0, 0, time.UTC)
|
||||
path, _, err := WriteFile(dir, sample(), at, "x")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if filepath.Base(path) != "minstrel-20261102T030000Z.tar" {
|
||||
t.Errorf("path = %s", path)
|
||||
}
|
||||
entries, _ := os.ReadDir(dir)
|
||||
if len(entries) != 1 || entries[0].Name() != filepath.Base(path) {
|
||||
t.Errorf("the folder holds %v, want only the backup", entries)
|
||||
}
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer f.Close()
|
||||
if _, _, err := Read(f); err != nil {
|
||||
t.Errorf("the written file doesn't read: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAFailedWriteFileLeavesNothingBehind(t *testing.T) {
|
||||
dir := filepath.Join(t.TempDir(), "missing")
|
||||
if _, _, err := WriteFile(dir, sample(), time.Now(), "x"); err == nil {
|
||||
t.Fatal("writing into a missing folder succeeded")
|
||||
}
|
||||
if _, err := os.Stat(dir); !os.IsNotExist(err) {
|
||||
t.Errorf("the failed write created %s", dir)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileNamesSortByTimeAndAreRecognised(t *testing.T) {
|
||||
earlier := FileName(time.Date(2026, 1, 9, 23, 0, 0, 0, time.UTC))
|
||||
later := FileName(time.Date(2026, 1, 10, 1, 0, 0, 0, time.FixedZone("x", 5*3600)))
|
||||
if earlier >= later {
|
||||
t.Errorf("%s does not sort before %s", earlier, later)
|
||||
}
|
||||
for name, want := range map[string]bool{
|
||||
earlier: true,
|
||||
"minstrel-20261102T030000Z.tar": true,
|
||||
".minstrel-20261102T030000Z.tar.part": false,
|
||||
"minstrel-yesterday.tar": false,
|
||||
"fablednotary-20261102T030000Z.tar": false,
|
||||
} {
|
||||
if IsFileName(name) != want {
|
||||
t.Errorf("IsFileName(%q) = %v, want %v", name, !want, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,201 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ErrS3 is a request the bucket refused or never answered.
|
||||
var ErrS3 = errors.New("backup: the S3 bucket refused the request")
|
||||
|
||||
// Bucket is a path-style S3 bucket: an endpoint such as Fabled Archives'
|
||||
// backup port or MinIO, a bucket name, and a key pair. A backup only ever
|
||||
// adds objects and lists them, so an append-only key is enough.
|
||||
type Bucket struct {
|
||||
Endpoint string
|
||||
Name string
|
||||
AccessKey string
|
||||
SecretKey string
|
||||
// Region defaults to us-east-1, which S3-compatible servers accept.
|
||||
Region string
|
||||
Client *http.Client
|
||||
// now is the signing clock; tests pin it.
|
||||
now func() time.Time
|
||||
}
|
||||
|
||||
// Object is one stored object.
|
||||
type Object struct {
|
||||
Key string
|
||||
Size int64
|
||||
LastModified time.Time
|
||||
}
|
||||
|
||||
func (b Bucket) region() string {
|
||||
if b.Region == "" {
|
||||
return "us-east-1"
|
||||
}
|
||||
return b.Region
|
||||
}
|
||||
|
||||
func (b Bucket) client() *http.Client {
|
||||
if b.Client != nil {
|
||||
return b.Client
|
||||
}
|
||||
return http.DefaultClient
|
||||
}
|
||||
|
||||
func (b Bucket) clock() time.Time {
|
||||
if b.now != nil {
|
||||
return b.now()
|
||||
}
|
||||
return time.Now()
|
||||
}
|
||||
|
||||
// objectURL is the path-style URL of key, encoded exactly as it is signed.
|
||||
func (b Bucket) objectURL(key string) (*url.URL, error) {
|
||||
u, err := url.Parse(strings.TrimRight(b.Endpoint, "/"))
|
||||
if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
|
||||
return nil, fmt.Errorf("%w: the endpoint %q is not an http(s) address", ErrS3, b.Endpoint)
|
||||
}
|
||||
if b.Name == "" {
|
||||
return nil, fmt.Errorf("%w: no bucket name", ErrS3)
|
||||
}
|
||||
raw := "/" + uriEncode(b.Name)
|
||||
if key != "" {
|
||||
for s := range strings.SplitSeq(key, "/") {
|
||||
raw += "/" + uriEncode(s)
|
||||
}
|
||||
}
|
||||
u.RawPath = u.EscapedPath() + raw
|
||||
u.Path, _ = url.PathUnescape(u.RawPath)
|
||||
return u, nil
|
||||
}
|
||||
|
||||
func (b Bucket) do(req *http.Request, payloadSHA256 string) (*http.Response, error) {
|
||||
sign(req, payloadSHA256, b.AccessKey, b.SecretKey, b.region(), b.clock())
|
||||
resp, err := b.client().Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %s is unreachable: %v", ErrS3, req.URL.Host, err)
|
||||
}
|
||||
if resp.StatusCode/100 != 2 {
|
||||
defer resp.Body.Close()
|
||||
return nil, refusal(resp)
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
// Put stores body under key. size is the body's length and payloadSHA256
|
||||
// its hex SHA-256, which the request is signed over.
|
||||
func (b Bucket) Put(ctx context.Context, key string, body io.Reader, size int64, payloadSHA256 string) error {
|
||||
u, err := b.objectURL(key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPut, u.String(), body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.URL = u
|
||||
req.ContentLength = size
|
||||
if size == 0 {
|
||||
req.Body = http.NoBody
|
||||
}
|
||||
resp, err := b.do(req, payloadSHA256)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
drain(resp)
|
||||
return nil
|
||||
}
|
||||
|
||||
// List returns up to limit objects whose keys start with prefix, in key
|
||||
// order.
|
||||
func (b Bucket) List(ctx context.Context, prefix string, limit int) ([]Object, error) {
|
||||
if limit < 1 {
|
||||
limit = 1000
|
||||
}
|
||||
var out []Object
|
||||
token := ""
|
||||
for {
|
||||
u, err := b.objectURL("")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
q := url.Values{"list-type": {"2"}, "prefix": {prefix}}
|
||||
if page := limit - len(out); page < 1000 {
|
||||
q.Set("max-keys", strconv.Itoa(page))
|
||||
}
|
||||
if token != "" {
|
||||
q.Set("continuation-token", token)
|
||||
}
|
||||
u.RawQuery = canonicalQuery(q)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.URL = u
|
||||
resp, err := b.do(req, emptySHA256)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var page struct {
|
||||
Contents []struct {
|
||||
Key string `xml:"Key"`
|
||||
Size int64 `xml:"Size"`
|
||||
LastModified string `xml:"LastModified"`
|
||||
} `xml:"Contents"`
|
||||
IsTruncated bool `xml:"IsTruncated"`
|
||||
NextContinuationToken string `xml:"NextContinuationToken"`
|
||||
}
|
||||
err = xml.NewDecoder(io.LimitReader(resp.Body, 16<<20)).Decode(&page)
|
||||
drain(resp)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: the listing is not S3 XML: %v", ErrS3, err)
|
||||
}
|
||||
for _, c := range page.Contents {
|
||||
modified, _ := time.Parse(time.RFC3339, c.LastModified)
|
||||
out = append(out, Object{Key: c.Key, Size: c.Size, LastModified: modified})
|
||||
}
|
||||
if !page.IsTruncated || page.NextContinuationToken == "" || len(out) >= limit {
|
||||
return out, nil
|
||||
}
|
||||
token = page.NextContinuationToken
|
||||
}
|
||||
}
|
||||
|
||||
// refusal turns an S3 error response into words an admin can act on.
|
||||
func refusal(resp *http.Response) error {
|
||||
var e struct {
|
||||
Code string `xml:"Code"`
|
||||
Message string `xml:"Message"`
|
||||
}
|
||||
_ = xml.NewDecoder(io.LimitReader(resp.Body, 64<<10)).Decode(&e)
|
||||
hint := map[string]string{
|
||||
"SignatureDoesNotMatch": "the secret key is wrong",
|
||||
"InvalidAccessKeyId": "the access key is not known to this server",
|
||||
"AccessDenied": "the key may not do this in this bucket",
|
||||
"NoSuchBucket": "there is no bucket by that name",
|
||||
"SlowDown": "the server asks to slow down",
|
||||
}[e.Code]
|
||||
switch {
|
||||
case hint != "":
|
||||
return fmt.Errorf("%w: HTTP %d %s: %s", ErrS3, resp.StatusCode, e.Code, hint)
|
||||
case e.Code != "":
|
||||
return fmt.Errorf("%w: HTTP %d %s: %s", ErrS3, resp.StatusCode, e.Code, e.Message)
|
||||
default:
|
||||
return fmt.Errorf("%w: HTTP %d", ErrS3, resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
func drain(resp *http.Response) {
|
||||
_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 1<<20))
|
||||
_ = resp.Body.Close()
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// AWS's published examples ("Examples: Signature Calculations in AWS
|
||||
// Signature Version 4", Amazon S3 API reference).
|
||||
const (
|
||||
awsAccessKey = "AKIAIOSFODNN7EXAMPLE"
|
||||
awsSecretKey = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
|
||||
awsCred = "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request"
|
||||
)
|
||||
|
||||
var awsTime = time.Date(2013, 5, 24, 0, 0, 0, 0, time.UTC)
|
||||
|
||||
func TestSignMatchesAWSsGetObjectExample(t *testing.T) {
|
||||
req, _ := http.NewRequest(http.MethodGet, "http://examplebucket.s3.amazonaws.com/test.txt", nil)
|
||||
req.Header.Set("Range", "bytes=0-9")
|
||||
sign(req, emptySHA256, awsAccessKey, awsSecretKey, "us-east-1", awsTime)
|
||||
want := awsCred + ", SignedHeaders=host;range;x-amz-content-sha256;x-amz-date, Signature=f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41"
|
||||
if got := req.Header.Get("Authorization"); got != want {
|
||||
t.Errorf("Authorization =\n%s\nwant\n%s", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSignMatchesAWSsPutObjectExample(t *testing.T) {
|
||||
req, _ := http.NewRequest(http.MethodPut, "http://examplebucket.s3.amazonaws.com/test$file.text", strings.NewReader("Welcome to Amazon S3."))
|
||||
req.Header.Set("Date", "Fri, 24 May 2013 00:00:00 GMT")
|
||||
req.Header.Set("X-Amz-Storage-Class", "REDUCED_REDUNDANCY")
|
||||
payload := hexSHA256([]byte("Welcome to Amazon S3."))
|
||||
if payload != "44ce7dd67c959e0d3524ffac1771dfbba87d2b6b4b4e99e42034a8b803f8b072" {
|
||||
t.Fatalf("payload hash = %s", payload)
|
||||
}
|
||||
sign(req, payload, awsAccessKey, awsSecretKey, "us-east-1", awsTime)
|
||||
want := awsCred + ", SignedHeaders=date;host;x-amz-content-sha256;x-amz-date;x-amz-storage-class, Signature=98ad721746da40c64f1a55b78f14c238d841ea1380cd77a1b5971af0ece108bd"
|
||||
if got := req.Header.Get("Authorization"); got != want {
|
||||
t.Errorf("Authorization =\n%s\nwant\n%s", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSignMatchesAWSsListObjectsExample(t *testing.T) {
|
||||
req, _ := http.NewRequest(http.MethodGet, "http://examplebucket.s3.amazonaws.com/?max-keys=2&prefix=J", nil)
|
||||
sign(req, emptySHA256, awsAccessKey, awsSecretKey, "us-east-1", awsTime)
|
||||
want := awsCred + ", SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=34b48302e7b5fa45bde8084f4b7868a86f0a534bc59db6670ed5711ef69dc6f7"
|
||||
if got := req.Header.Get("Authorization"); got != want {
|
||||
t.Errorf("Authorization =\n%s\nwant\n%s", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUriEncodeKeepsOnlyUnreservedCharacters(t *testing.T) {
|
||||
for in, want := range map[string]string{
|
||||
"01 - Song.flac": "01%20-%20Song.flac",
|
||||
"a+b=c&d": "a%2Bb%3Dc%26d",
|
||||
"Björk~_.": "Bj%C3%B6rk~_.",
|
||||
"a/b": "a%2Fb",
|
||||
} {
|
||||
if got := uriEncode(in); got != want {
|
||||
t.Errorf("uriEncode(%q) = %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func testBucket(url string) Bucket {
|
||||
return Bucket{Endpoint: url, Name: "minstrel", AccessKey: "ak", SecretKey: "sk",
|
||||
now: func() time.Time { return awsTime }}
|
||||
}
|
||||
|
||||
func TestPutSendsTheBodyToItsPathStyleKey(t *testing.T) {
|
||||
var gotPath, gotSHA, gotBody, gotAuth string
|
||||
var gotLen int64
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotPath, gotSHA, gotAuth, gotLen = r.URL.EscapedPath(), r.Header.Get("X-Amz-Content-Sha256"), r.Header.Get("Authorization"), r.ContentLength
|
||||
b, _ := io.ReadAll(r.Body)
|
||||
gotBody = string(b)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
body := "flac bytes"
|
||||
sum := hexSHA256([]byte(body))
|
||||
if err := testBucket(srv.URL).Put(context.Background(), "music/A B/01 + x.flac", strings.NewReader(body), int64(len(body)), sum); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if gotPath != "/minstrel/music/A%20B/01%20%2B%20x.flac" {
|
||||
t.Errorf("path = %s", gotPath)
|
||||
}
|
||||
if gotSHA != sum || gotBody != body || gotLen != int64(len(body)) {
|
||||
t.Errorf("sha %s body %q length %d", gotSHA, gotBody, gotLen)
|
||||
}
|
||||
if !strings.HasPrefix(gotAuth, "AWS4-HMAC-SHA256 Credential=ak/20130524/us-east-1/s3/aws4_request") {
|
||||
t.Errorf("Authorization = %s", gotAuth)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListFollowsContinuationTokens(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
q := r.URL.Query()
|
||||
if r.URL.Path != "/minstrel" || q.Get("list-type") != "2" || q.Get("prefix") != "database/" {
|
||||
http.Error(w, "unexpected "+r.URL.String(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if q.Get("continuation-token") == "" {
|
||||
fmt.Fprint(w, `<ListBucketResult><IsTruncated>true</IsTruncated><NextContinuationToken>next</NextContinuationToken>
|
||||
<Contents><Key>database/minstrel-20261101T030000Z.tar</Key><Size>10</Size><LastModified>2026-11-01T03:00:05.000Z</LastModified></Contents>
|
||||
<Contents><Key>database/minstrel-20261102T030000Z.tar</Key><Size>11</Size><LastModified>2026-11-02T03:00:05.000Z</LastModified></Contents>
|
||||
</ListBucketResult>`)
|
||||
return
|
||||
}
|
||||
fmt.Fprint(w, `<ListBucketResult><IsTruncated>false</IsTruncated>
|
||||
<Contents><Key>database/minstrel-20261103T030000Z.tar</Key><Size>12</Size><LastModified>2026-11-03T03:00:05.000Z</LastModified></Contents>
|
||||
</ListBucketResult>`)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
objs, err := testBucket(srv.URL).List(context.Background(), "database/", 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(objs) != 3 || objs[2].Key != "database/minstrel-20261103T030000Z.tar" || objs[1].Size != 11 ||
|
||||
!objs[0].LastModified.Equal(time.Date(2026, 11, 1, 3, 0, 5, 0, time.UTC)) {
|
||||
t.Errorf("objects = %+v", objs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestARefusalSaysWhatIsWrong(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
fmt.Fprint(w, `<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match</Message></Error>`)
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
_, err := testBucket(srv.URL).List(context.Background(), "", 1)
|
||||
if !errors.Is(err, ErrS3) || !strings.Contains(err.Error(), "HTTP 403 SignatureDoesNotMatch: the secret key is wrong") {
|
||||
t.Errorf("err = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnUnreachableOrMalformedEndpointIsAnS3Error(t *testing.T) {
|
||||
srv := httptest.NewServer(http.NotFoundHandler())
|
||||
srv.Close()
|
||||
for name, endpoint := range map[string]string{"closed": srv.URL, "not http": "ftp://archives", "empty": ""} {
|
||||
if _, err := testBucket(endpoint).List(context.Background(), "", 1); !errors.Is(err, ErrS3) {
|
||||
t.Errorf("%s: err = %v, want ErrS3", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// emptySHA256 is the payload hash of a request with no body.
|
||||
const emptySHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
||||
|
||||
// sign adds AWS Signature Version 4 headers for S3 to req: X-Amz-Date,
|
||||
// X-Amz-Content-Sha256 and Authorization. Every header already on the
|
||||
// request is signed, along with the host.
|
||||
func sign(req *http.Request, payloadSHA256, accessKey, secretKey, region string, at time.Time) {
|
||||
at = at.UTC()
|
||||
stamp := at.Format("20060102T150405Z")
|
||||
date := at.Format("20060102")
|
||||
req.Header.Set("X-Amz-Date", stamp)
|
||||
req.Header.Set("X-Amz-Content-Sha256", payloadSHA256)
|
||||
|
||||
host := req.Host
|
||||
if host == "" {
|
||||
host = req.URL.Host
|
||||
}
|
||||
headers := map[string]string{"host": host}
|
||||
for name, values := range req.Header {
|
||||
headers[strings.ToLower(name)] = strings.TrimSpace(strings.Join(values, ","))
|
||||
}
|
||||
names := make([]string, 0, len(headers))
|
||||
for name := range headers {
|
||||
names = append(names, name)
|
||||
}
|
||||
slices.Sort(names)
|
||||
var canonHeaders strings.Builder
|
||||
for _, name := range names {
|
||||
canonHeaders.WriteString(name + ":" + headers[name] + "\n")
|
||||
}
|
||||
signed := strings.Join(names, ";")
|
||||
|
||||
canonical := strings.Join([]string{
|
||||
req.Method,
|
||||
canonicalPath(req.URL),
|
||||
canonicalQuery(req.URL.Query()),
|
||||
canonHeaders.String(),
|
||||
signed,
|
||||
payloadSHA256,
|
||||
}, "\n")
|
||||
scope := date + "/" + region + "/s3/aws4_request"
|
||||
toSign := "AWS4-HMAC-SHA256\n" + stamp + "\n" + scope + "\n" + hexSHA256([]byte(canonical))
|
||||
|
||||
key := hmacSHA256([]byte("AWS4"+secretKey), date)
|
||||
key = hmacSHA256(key, region)
|
||||
key = hmacSHA256(key, "s3")
|
||||
key = hmacSHA256(key, "aws4_request")
|
||||
signature := hex.EncodeToString(hmacSHA256(key, toSign))
|
||||
|
||||
req.Header.Set("Authorization", "AWS4-HMAC-SHA256 Credential="+accessKey+"/"+scope+
|
||||
", SignedHeaders="+signed+", Signature="+signature)
|
||||
}
|
||||
|
||||
// canonicalPath is the request path with each segment URI-encoded once.
|
||||
func canonicalPath(u *url.URL) string {
|
||||
p := u.EscapedPath()
|
||||
if p == "" {
|
||||
return "/"
|
||||
}
|
||||
if unescaped, err := url.PathUnescape(p); err == nil {
|
||||
p = unescaped
|
||||
}
|
||||
segments := strings.Split(p, "/")
|
||||
for i, s := range segments {
|
||||
segments[i] = uriEncode(s)
|
||||
}
|
||||
return strings.Join(segments, "/")
|
||||
}
|
||||
|
||||
// canonicalQuery is the query sorted by name, names and values URI-encoded.
|
||||
func canonicalQuery(q url.Values) string {
|
||||
pairs := make([]string, 0, len(q))
|
||||
for name, values := range q {
|
||||
for _, v := range values {
|
||||
pairs = append(pairs, uriEncode(name)+"="+uriEncode(v))
|
||||
}
|
||||
}
|
||||
slices.Sort(pairs)
|
||||
return strings.Join(pairs, "&")
|
||||
}
|
||||
|
||||
// uriEncode percent-encodes everything but the unreserved characters, as
|
||||
// SigV4 requires; url.QueryEscape differs on space and '~'.
|
||||
func uriEncode(s string) string {
|
||||
var b strings.Builder
|
||||
for i := 0; i < len(s); i++ {
|
||||
c := s[i]
|
||||
if c >= 'A' && c <= 'Z' || c >= 'a' && c <= 'z' || c >= '0' && c <= '9' || c == '-' || c == '_' || c == '.' || c == '~' {
|
||||
b.WriteByte(c)
|
||||
continue
|
||||
}
|
||||
b.WriteString("%" + strings.ToUpper(hex.EncodeToString([]byte{c})))
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func hmacSHA256(key []byte, msg string) []byte {
|
||||
m := hmac.New(sha256.New, key)
|
||||
m.Write([]byte(msg))
|
||||
return m.Sum(nil)
|
||||
}
|
||||
|
||||
func hexSHA256(b []byte) string {
|
||||
sum := sha256.Sum256(b)
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
Reference in New Issue
Block a user