Compare commits

...
2 Commits
Author SHA1 Message Date
bvandeusenandClaude Opus 5.5 b6ae205862 feat: Minstrel can put and list objects in a path-style S3 bucket, signed with SigV4 and no SDK (#6771)
release / govulncheck (push) Successful in 14s
release / android (push) Successful in 4m37s
release / Build signed APK (push) Successful in 4m47s
release / integration (push) Failing after 5m3s
release / Attach APK to the Release (tag releases only) (push) Skipped
release / Build + push container image (push) Skipped
release / Verify release artifacts (tag releases only) (push) Skipped
release / go (push) Failing after 1m7s
release / web (push) Successful in 1m14s
Bucket.Put stores a body under a key, signed over its SHA-256; Bucket.List
pages through ListObjectsV2. A backup only adds and lists, so an
append-only key is enough. Keys are encoded once, exactly as signed, so
music paths with spaces, plus signs and accents reach the bucket intact.
A refusal names S3's code in words ("the secret key is wrong").

The signer is pinned to AWS's published GET Object, PUT Object and
ListObjects examples.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-02 13:51:02 -05:00
bvandeusenandClaude Opus 5.5 8be69b752a feat: a backup is one uncompressed tar of per-table JSON Lines that stays the same bytes day to day (#6769)
The file is minstrel-<UTC stamp>.tar: manifest.json first, then
tables/<table>.jsonl, rows one per line in primary-key order. The run's
time lives only in the manifest and member headers are fixed, so a
deduplicating off-site copy stores unchanged tables once.

WriteFile writes to a hidden .part file, syncs it and renames it, so a
watched folder never shows half a backup. Read refuses anything that
isn't a whole backup: another format, a newer format version, a cut-off
file, or a member holding fewer rows than the manifest says.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-02 13:51:02 -05:00
5 changed files with 890 additions and 0 deletions
+226
View File
@@ -0,0 +1,226 @@
package backup
import (
"archive/tar"
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"slices"
"strings"
"time"
)
// The backup file is laid out so that what didn't change between two days is
// the same bytes both times, which suits a deduplicating off-site copy such as
// Fabled Archives' restic:
//
// - Never compressed. Compression rewrites every byte after the first
// change, so no chunk would match again.
// - One member per table, tables/<table>.jsonl: one row per line, in
// primary-key order, keys sorted, no spacing.
// - What changes every run lives only in manifest.json, the first member.
// Member headers are fixed (mtime 0, owner 0, mode 0644).
// - A new dated name each run, so the target keeps a history and a sender
// only ever adds.
const (
FormatName = "minstrel-backup-tar"
FormatVersion = 1
manifestName = "manifest.json"
tablesDir = "tables/"
tableSuffix = ".jsonl"
namePrefix = "minstrel-"
nameSuffix = ".tar"
// maxMember bounds any one member a read takes in: far past the largest
// library's tracks table, small enough that a hostile file can't fill
// memory.
maxMember = 1 << 30
)
// Manifest is the file's first member.
type Manifest struct {
Format string `json:"format"`
FormatVersion int `json:"format_version"`
CreatedAt time.Time `json:"created_at"`
// Build is the Minstrel version that wrote the file.
Build string `json:"build"`
// Schema is the database's migration version when the file was written.
// A restore refuses a file from a newer schema than its own.
Schema int64 `json:"schema"`
// Rows is each table's row count, so a read can tell a whole member from
// a cut-off one.
Rows map[string]int `json:"rows"`
}
// FileName is a backup's file name: minstrel-20261102T030000Z.tar, in UTC, so
// names sort by time.
func FileName(at time.Time) string {
return namePrefix + at.UTC().Format("20060102T150405Z") + nameSuffix
}
// IsFileName reports whether a name is one FileName gives.
func IsFileName(name string) bool {
stamp, ok := strings.CutPrefix(name, namePrefix)
if !ok {
return false
}
stamp, ok = strings.CutSuffix(stamp, nameSuffix)
if !ok {
return false
}
_, err := time.Parse("20060102T150405Z", stamp)
return err == nil
}
// Write writes c as a backup file to w and returns its manifest.
func Write(w io.Writer, c Contents, at time.Time, build string) (Manifest, error) {
names := make([]string, 0, len(c.Tables))
m := Manifest{
Format: FormatName,
FormatVersion: FormatVersion,
CreatedAt: at.UTC(),
Build: build,
Schema: c.Schema,
Rows: make(map[string]int, len(c.Tables)),
}
for name, lines := range c.Tables {
names = append(names, name)
m.Rows[name] = len(lines)
}
slices.Sort(names)
manifest, err := json.MarshalIndent(m, "", " ")
if err != nil {
return m, err
}
tw := tar.NewWriter(w)
if err := member(tw, manifestName, append(manifest, '\n')); err != nil {
return m, err
}
for _, name := range names {
var body bytes.Buffer
for _, line := range c.Tables[name] {
body.Write(line)
body.WriteByte('\n')
}
if err := member(tw, tablesDir+name+tableSuffix, body.Bytes()); err != nil {
return m, err
}
}
return m, tw.Close()
}
// WriteFile writes c into dir under its dated name. The bytes go to a hidden
// .part file first, are synced, and only then take the backup's name, so
// anything watching the folder never sees half a backup under a backup's
// name, and a run that fails leaves nothing behind.
func WriteFile(dir string, c Contents, at time.Time, build string) (string, Manifest, error) {
final := filepath.Join(dir, FileName(at))
part := filepath.Join(dir, "."+FileName(at)+".part")
f, err := os.OpenFile(part, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600)
if err != nil {
return "", Manifest{}, fmt.Errorf("backup: create %s: %w", part, err)
}
m, err := Write(f, c, at, build)
if err == nil {
err = f.Sync()
}
if cerr := f.Close(); err == nil {
err = cerr
}
if err == nil {
err = os.Rename(part, final)
}
if err != nil {
_ = os.Remove(part)
return "", m, fmt.Errorf("backup: write %s: %w", final, err)
}
if d, err := os.Open(dir); err == nil {
_ = d.Sync()
_ = d.Close()
}
return final, m, nil
}
func member(tw *tar.Writer, name string, data []byte) error {
hdr := &tar.Header{
Name: name,
Mode: 0o644,
Size: int64(len(data)),
ModTime: time.Unix(0, 0),
Typeflag: tar.TypeReg,
Format: tar.FormatUSTAR,
}
if err := tw.WriteHeader(hdr); err != nil {
return err
}
_, err := tw.Write(data)
return err
}
// Read reads a backup file, refusing anything that isn't a whole one.
func Read(r io.Reader) (Manifest, Contents, error) {
var m Manifest
c := Contents{Tables: map[string][][]byte{}}
tr := tar.NewReader(r)
first := true
for {
hdr, err := tr.Next()
if errors.Is(err, io.EOF) {
break
}
if err != nil {
return m, c, fmt.Errorf("%w: %v", ErrNotABackup, err)
}
if hdr.Size > maxMember {
return m, c, fmt.Errorf("%w: %s is too large", ErrNotABackup, hdr.Name)
}
data, err := io.ReadAll(io.LimitReader(tr, maxMember))
if err != nil {
return m, c, fmt.Errorf("%w: %v", ErrNotABackup, err)
}
if first {
first = false
if hdr.Name != manifestName || json.Unmarshal(data, &m) != nil || m.Format != FormatName {
return m, c, ErrNotABackup
}
if m.FormatVersion > FormatVersion {
return m, c, fmt.Errorf("%w: it was written by a newer Minstrel (format %d)", ErrNotABackup, m.FormatVersion)
}
c.Schema = m.Schema
continue
}
table, ok := strings.CutPrefix(hdr.Name, tablesDir)
if !ok {
continue
}
if table, ok = strings.CutSuffix(table, tableSuffix); !ok || table == "" {
continue
}
var lines [][]byte
for line := range bytes.SplitSeq(data, []byte("\n")) {
if len(line) == 0 {
continue
}
if !json.Valid(line) {
return m, c, fmt.Errorf("%w: a row in %s is not JSON", ErrNotABackup, table)
}
lines = append(lines, line)
}
c.Tables[table] = lines
}
if first {
return m, c, ErrNotABackup
}
for table, want := range m.Rows {
if got := len(c.Tables[table]); got != want {
return m, c, fmt.Errorf("%w: %s holds %d rows, the manifest says %d", ErrNotABackup, table, got, want)
}
}
return m, c, nil
}
+190
View File
@@ -0,0 +1,190 @@
package backup
import (
"archive/tar"
"bytes"
"errors"
"io"
"os"
"path/filepath"
"testing"
"time"
)
func sample() Contents {
return Contents{
Schema: 98,
Tables: map[string][][]byte{
"users": {[]byte(`{"id":"u1","username":"ada"}`), []byte(`{"id":"u2","username":"bo"}`)},
"tracks": {[]byte(`{"file_path":"/music/a.flac","id":"t1"}`)},
"wants": {},
},
}
}
func members(t *testing.T, file []byte) map[string][]byte {
t.Helper()
out := map[string][]byte{}
tr := tar.NewReader(bytes.NewReader(file))
for {
hdr, err := tr.Next()
if errors.Is(err, io.EOF) {
return out
}
if err != nil {
t.Fatal(err)
}
data, _ := io.ReadAll(tr)
out[hdr.Name] = data
}
}
func TestABackupFileReadsBackAsWritten(t *testing.T) {
var buf bytes.Buffer
at := time.Date(2026, 11, 2, 3, 0, 0, 0, time.UTC)
if _, err := Write(&buf, sample(), at, "2026.11.02.0300"); err != nil {
t.Fatal(err)
}
m, c, err := Read(&buf)
if err != nil {
t.Fatal(err)
}
if m.Schema != 98 || c.Schema != 98 || m.Build != "2026.11.02.0300" || !m.CreatedAt.Equal(at) {
t.Errorf("manifest = %+v", m)
}
want := sample()
for name, lines := range want.Tables {
got := c.Tables[name]
if len(got) != len(lines) {
t.Errorf("%s: %d rows, want %d", name, len(got), len(lines))
continue
}
for i := range lines {
if !bytes.Equal(got[i], lines[i]) {
t.Errorf("%s row %d = %s, want %s", name, i, got[i], lines[i])
}
}
}
}
// Two days over unchanged data differ only in the manifest, so a
// deduplicating copy stores the tables once.
func TestOnlyTheManifestChangesBetweenDays(t *testing.T) {
var day1, day2 bytes.Buffer
if _, err := Write(&day1, sample(), time.Date(2026, 11, 1, 3, 0, 0, 0, time.UTC), "a"); err != nil {
t.Fatal(err)
}
if _, err := Write(&day2, sample(), time.Date(2026, 11, 2, 3, 0, 0, 0, time.UTC), "b"); err != nil {
t.Fatal(err)
}
a, b := members(t, day1.Bytes()), members(t, day2.Bytes())
if len(a) != 4 {
t.Fatalf("members = %d, want the manifest and three tables", len(a))
}
for name, data := range a {
same := bytes.Equal(data, b[name])
if name == manifestName && same {
t.Error("the manifest is identical across days; it should carry the run's time")
}
if name != manifestName && !same {
t.Errorf("%s differs between days over the same rows", name)
}
}
// The table members' bytes, headers included, sit unchanged in both files.
tracks := a[tablesDir+"tracks"+tableSuffix]
if !bytes.Contains(day1.Bytes(), tracks) || !bytes.Contains(day2.Bytes(), tracks) {
t.Error("the tracks member is not stored verbatim")
}
}
func TestReadRefusesWhatIsNotAWholeBackup(t *testing.T) {
var good bytes.Buffer
if _, err := Write(&good, sample(), time.Now(), "x"); err != nil {
t.Fatal(err)
}
tarOf := func(name string, data []byte) []byte {
var buf bytes.Buffer
tw := tar.NewWriter(&buf)
_ = member(tw, name, data)
_ = tw.Close()
return buf.Bytes()
}
short := func() []byte {
// The manifest promises two users; the member holds one.
var buf bytes.Buffer
tw := tar.NewWriter(&buf)
_ = member(tw, manifestName, []byte(`{"format":"minstrel-backup-tar","format_version":1,"rows":{"users":2}}`))
_ = member(tw, tablesDir+"users"+tableSuffix, []byte(`{"id":"u1"}`+"\n"))
_ = tw.Close()
return buf.Bytes()
}
cases := map[string][]byte{
"empty": nil,
"not a tar": []byte("hello"),
"cut off": good.Bytes()[:good.Len()/2],
"no manifest": tarOf(tablesDir+"users"+tableSuffix, []byte(`{"id":"u1"}`)),
"another format": tarOf(manifestName, []byte(`{"format":"fablednotary-backup-tar","format_version":1}`)),
"a newer format": tarOf(manifestName, []byte(`{"format":"minstrel-backup-tar","format_version":99}`)),
"a member short": short(),
}
for name, file := range cases {
if _, _, err := Read(bytes.NewReader(file)); !errors.Is(err, ErrNotABackup) {
t.Errorf("%s: err = %v, want ErrNotABackup", name, err)
}
}
}
func TestWriteFileLandsUnderTheDatedNameOnly(t *testing.T) {
dir := t.TempDir()
at := time.Date(2026, 11, 2, 3, 0, 0, 0, time.UTC)
path, _, err := WriteFile(dir, sample(), at, "x")
if err != nil {
t.Fatal(err)
}
if filepath.Base(path) != "minstrel-20261102T030000Z.tar" {
t.Errorf("path = %s", path)
}
entries, _ := os.ReadDir(dir)
if len(entries) != 1 || entries[0].Name() != filepath.Base(path) {
t.Errorf("the folder holds %v, want only the backup", entries)
}
f, err := os.Open(path)
if err != nil {
t.Fatal(err)
}
defer f.Close()
if _, _, err := Read(f); err != nil {
t.Errorf("the written file doesn't read: %v", err)
}
}
func TestAFailedWriteFileLeavesNothingBehind(t *testing.T) {
dir := filepath.Join(t.TempDir(), "missing")
if _, _, err := WriteFile(dir, sample(), time.Now(), "x"); err == nil {
t.Fatal("writing into a missing folder succeeded")
}
if _, err := os.Stat(dir); !os.IsNotExist(err) {
t.Errorf("the failed write created %s", dir)
}
}
func TestFileNamesSortByTimeAndAreRecognised(t *testing.T) {
earlier := FileName(time.Date(2026, 1, 9, 23, 0, 0, 0, time.UTC))
later := FileName(time.Date(2026, 1, 10, 1, 0, 0, 0, time.FixedZone("x", 5*3600)))
if earlier >= later {
t.Errorf("%s does not sort before %s", earlier, later)
}
for name, want := range map[string]bool{
earlier: true,
"minstrel-20261102T030000Z.tar": true,
".minstrel-20261102T030000Z.tar.part": false,
"minstrel-yesterday.tar": false,
"fablednotary-20261102T030000Z.tar": false,
} {
if IsFileName(name) != want {
t.Errorf("IsFileName(%q) = %v, want %v", name, !want, want)
}
}
}
+201
View File
@@ -0,0 +1,201 @@
package backup
import (
"context"
"encoding/xml"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"strconv"
"strings"
"time"
)
// ErrS3 is a request the bucket refused or never answered.
var ErrS3 = errors.New("backup: the S3 bucket refused the request")
// Bucket is a path-style S3 bucket: an endpoint such as Fabled Archives'
// backup port or MinIO, a bucket name, and a key pair. A backup only ever
// adds objects and lists them, so an append-only key is enough.
type Bucket struct {
Endpoint string
Name string
AccessKey string
SecretKey string
// Region defaults to us-east-1, which S3-compatible servers accept.
Region string
Client *http.Client
// now is the signing clock; tests pin it.
now func() time.Time
}
// Object is one stored object.
type Object struct {
Key string
Size int64
LastModified time.Time
}
func (b Bucket) region() string {
if b.Region == "" {
return "us-east-1"
}
return b.Region
}
func (b Bucket) client() *http.Client {
if b.Client != nil {
return b.Client
}
return http.DefaultClient
}
func (b Bucket) clock() time.Time {
if b.now != nil {
return b.now()
}
return time.Now()
}
// objectURL is the path-style URL of key, encoded exactly as it is signed.
func (b Bucket) objectURL(key string) (*url.URL, error) {
u, err := url.Parse(strings.TrimRight(b.Endpoint, "/"))
if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
return nil, fmt.Errorf("%w: the endpoint %q is not an http(s) address", ErrS3, b.Endpoint)
}
if b.Name == "" {
return nil, fmt.Errorf("%w: no bucket name", ErrS3)
}
raw := "/" + uriEncode(b.Name)
if key != "" {
for s := range strings.SplitSeq(key, "/") {
raw += "/" + uriEncode(s)
}
}
u.RawPath = u.EscapedPath() + raw
u.Path, _ = url.PathUnescape(u.RawPath)
return u, nil
}
func (b Bucket) do(req *http.Request, payloadSHA256 string) (*http.Response, error) {
sign(req, payloadSHA256, b.AccessKey, b.SecretKey, b.region(), b.clock())
resp, err := b.client().Do(req)
if err != nil {
return nil, fmt.Errorf("%w: %s is unreachable: %v", ErrS3, req.URL.Host, err)
}
if resp.StatusCode/100 != 2 {
defer resp.Body.Close()
return nil, refusal(resp)
}
return resp, nil
}
// Put stores body under key. size is the body's length and payloadSHA256
// its hex SHA-256, which the request is signed over.
func (b Bucket) Put(ctx context.Context, key string, body io.Reader, size int64, payloadSHA256 string) error {
u, err := b.objectURL(key)
if err != nil {
return err
}
req, err := http.NewRequestWithContext(ctx, http.MethodPut, u.String(), body)
if err != nil {
return err
}
req.URL = u
req.ContentLength = size
if size == 0 {
req.Body = http.NoBody
}
resp, err := b.do(req, payloadSHA256)
if err != nil {
return err
}
drain(resp)
return nil
}
// List returns up to limit objects whose keys start with prefix, in key
// order.
func (b Bucket) List(ctx context.Context, prefix string, limit int) ([]Object, error) {
if limit < 1 {
limit = 1000
}
var out []Object
token := ""
for {
u, err := b.objectURL("")
if err != nil {
return nil, err
}
q := url.Values{"list-type": {"2"}, "prefix": {prefix}}
if page := limit - len(out); page < 1000 {
q.Set("max-keys", strconv.Itoa(page))
}
if token != "" {
q.Set("continuation-token", token)
}
u.RawQuery = canonicalQuery(q)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
if err != nil {
return nil, err
}
req.URL = u
resp, err := b.do(req, emptySHA256)
if err != nil {
return nil, err
}
var page struct {
Contents []struct {
Key string `xml:"Key"`
Size int64 `xml:"Size"`
LastModified string `xml:"LastModified"`
} `xml:"Contents"`
IsTruncated bool `xml:"IsTruncated"`
NextContinuationToken string `xml:"NextContinuationToken"`
}
err = xml.NewDecoder(io.LimitReader(resp.Body, 16<<20)).Decode(&page)
drain(resp)
if err != nil {
return nil, fmt.Errorf("%w: the listing is not S3 XML: %v", ErrS3, err)
}
for _, c := range page.Contents {
modified, _ := time.Parse(time.RFC3339, c.LastModified)
out = append(out, Object{Key: c.Key, Size: c.Size, LastModified: modified})
}
if !page.IsTruncated || page.NextContinuationToken == "" || len(out) >= limit {
return out, nil
}
token = page.NextContinuationToken
}
}
// refusal turns an S3 error response into words an admin can act on.
func refusal(resp *http.Response) error {
var e struct {
Code string `xml:"Code"`
Message string `xml:"Message"`
}
_ = xml.NewDecoder(io.LimitReader(resp.Body, 64<<10)).Decode(&e)
hint := map[string]string{
"SignatureDoesNotMatch": "the secret key is wrong",
"InvalidAccessKeyId": "the access key is not known to this server",
"AccessDenied": "the key may not do this in this bucket",
"NoSuchBucket": "there is no bucket by that name",
"SlowDown": "the server asks to slow down",
}[e.Code]
switch {
case hint != "":
return fmt.Errorf("%w: HTTP %d %s: %s", ErrS3, resp.StatusCode, e.Code, hint)
case e.Code != "":
return fmt.Errorf("%w: HTTP %d %s: %s", ErrS3, resp.StatusCode, e.Code, e.Message)
default:
return fmt.Errorf("%w: HTTP %d", ErrS3, resp.StatusCode)
}
}
func drain(resp *http.Response) {
_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 1<<20))
_ = resp.Body.Close()
}
+154
View File
@@ -0,0 +1,154 @@
package backup
import (
"context"
"errors"
"fmt"
"io"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
)
// AWS's published examples ("Examples: Signature Calculations in AWS
// Signature Version 4", Amazon S3 API reference).
const (
awsAccessKey = "AKIAIOSFODNN7EXAMPLE"
awsSecretKey = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
awsCred = "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request"
)
var awsTime = time.Date(2013, 5, 24, 0, 0, 0, 0, time.UTC)
func TestSignMatchesAWSsGetObjectExample(t *testing.T) {
req, _ := http.NewRequest(http.MethodGet, "http://examplebucket.s3.amazonaws.com/test.txt", nil)
req.Header.Set("Range", "bytes=0-9")
sign(req, emptySHA256, awsAccessKey, awsSecretKey, "us-east-1", awsTime)
want := awsCred + ", SignedHeaders=host;range;x-amz-content-sha256;x-amz-date, Signature=f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41"
if got := req.Header.Get("Authorization"); got != want {
t.Errorf("Authorization =\n%s\nwant\n%s", got, want)
}
}
func TestSignMatchesAWSsPutObjectExample(t *testing.T) {
req, _ := http.NewRequest(http.MethodPut, "http://examplebucket.s3.amazonaws.com/test$file.text", strings.NewReader("Welcome to Amazon S3."))
req.Header.Set("Date", "Fri, 24 May 2013 00:00:00 GMT")
req.Header.Set("X-Amz-Storage-Class", "REDUCED_REDUNDANCY")
payload := hexSHA256([]byte("Welcome to Amazon S3."))
if payload != "44ce7dd67c959e0d3524ffac1771dfbba87d2b6b4b4e99e42034a8b803f8b072" {
t.Fatalf("payload hash = %s", payload)
}
sign(req, payload, awsAccessKey, awsSecretKey, "us-east-1", awsTime)
want := awsCred + ", SignedHeaders=date;host;x-amz-content-sha256;x-amz-date;x-amz-storage-class, Signature=98ad721746da40c64f1a55b78f14c238d841ea1380cd77a1b5971af0ece108bd"
if got := req.Header.Get("Authorization"); got != want {
t.Errorf("Authorization =\n%s\nwant\n%s", got, want)
}
}
func TestSignMatchesAWSsListObjectsExample(t *testing.T) {
req, _ := http.NewRequest(http.MethodGet, "http://examplebucket.s3.amazonaws.com/?max-keys=2&prefix=J", nil)
sign(req, emptySHA256, awsAccessKey, awsSecretKey, "us-east-1", awsTime)
want := awsCred + ", SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=34b48302e7b5fa45bde8084f4b7868a86f0a534bc59db6670ed5711ef69dc6f7"
if got := req.Header.Get("Authorization"); got != want {
t.Errorf("Authorization =\n%s\nwant\n%s", got, want)
}
}
func TestUriEncodeKeepsOnlyUnreservedCharacters(t *testing.T) {
for in, want := range map[string]string{
"01 - Song.flac": "01%20-%20Song.flac",
"a+b=c&d": "a%2Bb%3Dc%26d",
"Björk~_.": "Bj%C3%B6rk~_.",
"a/b": "a%2Fb",
} {
if got := uriEncode(in); got != want {
t.Errorf("uriEncode(%q) = %q, want %q", in, got, want)
}
}
}
func testBucket(url string) Bucket {
return Bucket{Endpoint: url, Name: "minstrel", AccessKey: "ak", SecretKey: "sk",
now: func() time.Time { return awsTime }}
}
func TestPutSendsTheBodyToItsPathStyleKey(t *testing.T) {
var gotPath, gotSHA, gotBody, gotAuth string
var gotLen int64
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotPath, gotSHA, gotAuth, gotLen = r.URL.EscapedPath(), r.Header.Get("X-Amz-Content-Sha256"), r.Header.Get("Authorization"), r.ContentLength
b, _ := io.ReadAll(r.Body)
gotBody = string(b)
}))
defer srv.Close()
body := "flac bytes"
sum := hexSHA256([]byte(body))
if err := testBucket(srv.URL).Put(context.Background(), "music/A B/01 + x.flac", strings.NewReader(body), int64(len(body)), sum); err != nil {
t.Fatal(err)
}
if gotPath != "/minstrel/music/A%20B/01%20%2B%20x.flac" {
t.Errorf("path = %s", gotPath)
}
if gotSHA != sum || gotBody != body || gotLen != int64(len(body)) {
t.Errorf("sha %s body %q length %d", gotSHA, gotBody, gotLen)
}
if !strings.HasPrefix(gotAuth, "AWS4-HMAC-SHA256 Credential=ak/20130524/us-east-1/s3/aws4_request") {
t.Errorf("Authorization = %s", gotAuth)
}
}
func TestListFollowsContinuationTokens(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
q := r.URL.Query()
if r.URL.Path != "/minstrel" || q.Get("list-type") != "2" || q.Get("prefix") != "database/" {
http.Error(w, "unexpected "+r.URL.String(), http.StatusBadRequest)
return
}
if q.Get("continuation-token") == "" {
fmt.Fprint(w, `<ListBucketResult><IsTruncated>true</IsTruncated><NextContinuationToken>next</NextContinuationToken>
<Contents><Key>database/minstrel-20261101T030000Z.tar</Key><Size>10</Size><LastModified>2026-11-01T03:00:05.000Z</LastModified></Contents>
<Contents><Key>database/minstrel-20261102T030000Z.tar</Key><Size>11</Size><LastModified>2026-11-02T03:00:05.000Z</LastModified></Contents>
</ListBucketResult>`)
return
}
fmt.Fprint(w, `<ListBucketResult><IsTruncated>false</IsTruncated>
<Contents><Key>database/minstrel-20261103T030000Z.tar</Key><Size>12</Size><LastModified>2026-11-03T03:00:05.000Z</LastModified></Contents>
</ListBucketResult>`)
}))
defer srv.Close()
objs, err := testBucket(srv.URL).List(context.Background(), "database/", 0)
if err != nil {
t.Fatal(err)
}
if len(objs) != 3 || objs[2].Key != "database/minstrel-20261103T030000Z.tar" || objs[1].Size != 11 ||
!objs[0].LastModified.Equal(time.Date(2026, 11, 1, 3, 0, 5, 0, time.UTC)) {
t.Errorf("objects = %+v", objs)
}
}
func TestARefusalSaysWhatIsWrong(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusForbidden)
fmt.Fprint(w, `<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match</Message></Error>`)
}))
defer srv.Close()
_, err := testBucket(srv.URL).List(context.Background(), "", 1)
if !errors.Is(err, ErrS3) || !strings.Contains(err.Error(), "HTTP 403 SignatureDoesNotMatch: the secret key is wrong") {
t.Errorf("err = %v", err)
}
}
func TestAnUnreachableOrMalformedEndpointIsAnS3Error(t *testing.T) {
srv := httptest.NewServer(http.NotFoundHandler())
srv.Close()
for name, endpoint := range map[string]string{"closed": srv.URL, "not http": "ftp://archives", "empty": ""} {
if _, err := testBucket(endpoint).List(context.Background(), "", 1); !errors.Is(err, ErrS3) {
t.Errorf("%s: err = %v, want ErrS3", name, err)
}
}
}
+119
View File
@@ -0,0 +1,119 @@
package backup
import (
"crypto/hmac"
"crypto/sha256"
"encoding/hex"
"net/http"
"net/url"
"slices"
"strings"
"time"
)
// emptySHA256 is the payload hash of a request with no body.
const emptySHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
// sign adds AWS Signature Version 4 headers for S3 to req: X-Amz-Date,
// X-Amz-Content-Sha256 and Authorization. Every header already on the
// request is signed, along with the host.
func sign(req *http.Request, payloadSHA256, accessKey, secretKey, region string, at time.Time) {
at = at.UTC()
stamp := at.Format("20060102T150405Z")
date := at.Format("20060102")
req.Header.Set("X-Amz-Date", stamp)
req.Header.Set("X-Amz-Content-Sha256", payloadSHA256)
host := req.Host
if host == "" {
host = req.URL.Host
}
headers := map[string]string{"host": host}
for name, values := range req.Header {
headers[strings.ToLower(name)] = strings.TrimSpace(strings.Join(values, ","))
}
names := make([]string, 0, len(headers))
for name := range headers {
names = append(names, name)
}
slices.Sort(names)
var canonHeaders strings.Builder
for _, name := range names {
canonHeaders.WriteString(name + ":" + headers[name] + "\n")
}
signed := strings.Join(names, ";")
canonical := strings.Join([]string{
req.Method,
canonicalPath(req.URL),
canonicalQuery(req.URL.Query()),
canonHeaders.String(),
signed,
payloadSHA256,
}, "\n")
scope := date + "/" + region + "/s3/aws4_request"
toSign := "AWS4-HMAC-SHA256\n" + stamp + "\n" + scope + "\n" + hexSHA256([]byte(canonical))
key := hmacSHA256([]byte("AWS4"+secretKey), date)
key = hmacSHA256(key, region)
key = hmacSHA256(key, "s3")
key = hmacSHA256(key, "aws4_request")
signature := hex.EncodeToString(hmacSHA256(key, toSign))
req.Header.Set("Authorization", "AWS4-HMAC-SHA256 Credential="+accessKey+"/"+scope+
", SignedHeaders="+signed+", Signature="+signature)
}
// canonicalPath is the request path with each segment URI-encoded once.
func canonicalPath(u *url.URL) string {
p := u.EscapedPath()
if p == "" {
return "/"
}
if unescaped, err := url.PathUnescape(p); err == nil {
p = unescaped
}
segments := strings.Split(p, "/")
for i, s := range segments {
segments[i] = uriEncode(s)
}
return strings.Join(segments, "/")
}
// canonicalQuery is the query sorted by name, names and values URI-encoded.
func canonicalQuery(q url.Values) string {
pairs := make([]string, 0, len(q))
for name, values := range q {
for _, v := range values {
pairs = append(pairs, uriEncode(name)+"="+uriEncode(v))
}
}
slices.Sort(pairs)
return strings.Join(pairs, "&")
}
// uriEncode percent-encodes everything but the unreserved characters, as
// SigV4 requires; url.QueryEscape differs on space and '~'.
func uriEncode(s string) string {
var b strings.Builder
for i := 0; i < len(s); i++ {
c := s[i]
if c >= 'A' && c <= 'Z' || c >= 'a' && c <= 'z' || c >= '0' && c <= '9' || c == '-' || c == '_' || c == '.' || c == '~' {
b.WriteByte(c)
continue
}
b.WriteString("%" + strings.ToUpper(hex.EncodeToString([]byte{c})))
}
return b.String()
}
func hmacSHA256(key []byte, msg string) []byte {
m := hmac.New(sha256.New, key)
m.Write([]byte(msg))
return m.Sum(nil)
}
func hexSHA256(b []byte) string {
sum := sha256.Sum256(b)
return hex.EncodeToString(sum[:])
}