feat: Minstrel can put and list objects in a path-style S3 bucket, signed with SigV4 and no SDK (#6771)

Bucket.Put stores a body under a key, signed over its SHA-256; Bucket.List
pages through ListObjectsV2. A backup only adds and lists, so an
append-only key is enough. Keys are encoded once, exactly as signed, so
music paths with spaces, plus signs and accents reach the bucket intact.
A refusal names S3's code in words ("the secret key is wrong").

The signer is pinned to AWS's published GET Object, PUT Object and
ListObjects examples.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/backup/s3.go

@@ -0,0 +1,201 @@
+package backup
+
+import (
+	"context"
+	"encoding/xml"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// ErrS3 is a request the bucket refused or never answered.
+var ErrS3 = errors.New("backup: the S3 bucket refused the request")
+
+// Bucket is a path-style S3 bucket: an endpoint such as Fabled Archives'
+// backup port or MinIO, a bucket name, and a key pair. A backup only ever
+// adds objects and lists them, so an append-only key is enough.
+type Bucket struct {
+	Endpoint  string
+	Name      string
+	AccessKey string
+	SecretKey string
+	// Region defaults to us-east-1, which S3-compatible servers accept.
+	Region string
+	Client *http.Client
+	// now is the signing clock; tests pin it.
+	now func() time.Time
+}
+
+// Object is one stored object.
+type Object struct {
+	Key          string
+	Size         int64
+	LastModified time.Time
+}
+
+func (b Bucket) region() string {
+	if b.Region == "" {
+		return "us-east-1"
+	}
+	return b.Region
+}
+
+func (b Bucket) client() *http.Client {
+	if b.Client != nil {
+		return b.Client
+	}
+	return http.DefaultClient
+}
+
+func (b Bucket) clock() time.Time {
+	if b.now != nil {
+		return b.now()
+	}
+	return time.Now()
+}
+
+// objectURL is the path-style URL of key, encoded exactly as it is signed.
+func (b Bucket) objectURL(key string) (*url.URL, error) {
+	u, err := url.Parse(strings.TrimRight(b.Endpoint, "/"))
+	if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
+		return nil, fmt.Errorf("%w: the endpoint %q is not an http(s) address", ErrS3, b.Endpoint)
+	}
+	if b.Name == "" {
+		return nil, fmt.Errorf("%w: no bucket name", ErrS3)
+	}
+	raw := "/" + uriEncode(b.Name)
+	if key != "" {
+		for s := range strings.SplitSeq(key, "/") {
+			raw += "/" + uriEncode(s)
+		}
+	}
+	u.RawPath = u.EscapedPath() + raw
+	u.Path, _ = url.PathUnescape(u.RawPath)
+	return u, nil
+}
+
+func (b Bucket) do(req *http.Request, payloadSHA256 string) (*http.Response, error) {
+	sign(req, payloadSHA256, b.AccessKey, b.SecretKey, b.region(), b.clock())
+	resp, err := b.client().Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("%w: %s is unreachable: %v", ErrS3, req.URL.Host, err)
+	}
+	if resp.StatusCode/100 != 2 {
+		defer resp.Body.Close()
+		return nil, refusal(resp)
+	}
+	return resp, nil
+}
+
+// Put stores body under key. size is the body's length and payloadSHA256
+// its hex SHA-256, which the request is signed over.
+func (b Bucket) Put(ctx context.Context, key string, body io.Reader, size int64, payloadSHA256 string) error {
+	u, err := b.objectURL(key)
+	if err != nil {
+		return err
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodPut, u.String(), body)
+	if err != nil {
+		return err
+	}
+	req.URL = u
+	req.ContentLength = size
+	if size == 0 {
+		req.Body = http.NoBody
+	}
+	resp, err := b.do(req, payloadSHA256)
+	if err != nil {
+		return err
+	}
+	drain(resp)
+	return nil
+}
+
+// List returns up to limit objects whose keys start with prefix, in key
+// order.
+func (b Bucket) List(ctx context.Context, prefix string, limit int) ([]Object, error) {
+	if limit < 1 {
+		limit = 1000
+	}
+	var out []Object
+	token := ""
+	for {
+		u, err := b.objectURL("")
+		if err != nil {
+			return nil, err
+		}
+		q := url.Values{"list-type": {"2"}, "prefix": {prefix}}
+		if page := limit - len(out); page < 1000 {
+			q.Set("max-keys", strconv.Itoa(page))
+		}
+		if token != "" {
+			q.Set("continuation-token", token)
+		}
+		u.RawQuery = canonicalQuery(q)
+		req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
+		if err != nil {
+			return nil, err
+		}
+		req.URL = u
+		resp, err := b.do(req, emptySHA256)
+		if err != nil {
+			return nil, err
+		}
+		var page struct {
+			Contents []struct {
+				Key          string `xml:"Key"`
+				Size         int64  `xml:"Size"`
+				LastModified string `xml:"LastModified"`
+			} `xml:"Contents"`
+			IsTruncated           bool   `xml:"IsTruncated"`
+			NextContinuationToken string `xml:"NextContinuationToken"`
+		}
+		err = xml.NewDecoder(io.LimitReader(resp.Body, 16<<20)).Decode(&page)
+		drain(resp)
+		if err != nil {
+			return nil, fmt.Errorf("%w: the listing is not S3 XML: %v", ErrS3, err)
+		}
+		for _, c := range page.Contents {
+			modified, _ := time.Parse(time.RFC3339, c.LastModified)
+			out = append(out, Object{Key: c.Key, Size: c.Size, LastModified: modified})
+		}
+		if !page.IsTruncated || page.NextContinuationToken == "" || len(out) >= limit {
+			return out, nil
+		}
+		token = page.NextContinuationToken
+	}
+}
+
+// refusal turns an S3 error response into words an admin can act on.
+func refusal(resp *http.Response) error {
+	var e struct {
+		Code    string `xml:"Code"`
+		Message string `xml:"Message"`
+	}
+	_ = xml.NewDecoder(io.LimitReader(resp.Body, 64<<10)).Decode(&e)
+	hint := map[string]string{
+		"SignatureDoesNotMatch": "the secret key is wrong",
+		"InvalidAccessKeyId":    "the access key is not known to this server",
+		"AccessDenied":          "the key may not do this in this bucket",
+		"NoSuchBucket":          "there is no bucket by that name",
+		"SlowDown":              "the server asks to slow down",
+	}[e.Code]
+	switch {
+	case hint != "":
+		return fmt.Errorf("%w: HTTP %d %s: %s", ErrS3, resp.StatusCode, e.Code, hint)
+	case e.Code != "":
+		return fmt.Errorf("%w: HTTP %d %s: %s", ErrS3, resp.StatusCode, e.Code, e.Message)
+	default:
+		return fmt.Errorf("%w: HTTP %d", ErrS3, resp.StatusCode)
+	}
+}
+
+func drain(resp *http.Response) {
+	_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 1<<20))
+	_ = resp.Body.Close()
+}

internal/backup/s3_test.go

@@ -0,0 +1,154 @@
+package backup
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+)
+
+// AWS's published examples ("Examples: Signature Calculations in AWS
+// Signature Version 4", Amazon S3 API reference).
+const (
+	awsAccessKey = "AKIAIOSFODNN7EXAMPLE"
+	awsSecretKey = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
+	awsCred      = "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request"
+)
+
+var awsTime = time.Date(2013, 5, 24, 0, 0, 0, 0, time.UTC)
+
+func TestSignMatchesAWSsGetObjectExample(t *testing.T) {
+	req, _ := http.NewRequest(http.MethodGet, "http://examplebucket.s3.amazonaws.com/test.txt", nil)
+	req.Header.Set("Range", "bytes=0-9")
+	sign(req, emptySHA256, awsAccessKey, awsSecretKey, "us-east-1", awsTime)
+	want := awsCred + ", SignedHeaders=host;range;x-amz-content-sha256;x-amz-date, Signature=f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41"
+	if got := req.Header.Get("Authorization"); got != want {
+		t.Errorf("Authorization =\n%s\nwant\n%s", got, want)
+	}
+}
+
+func TestSignMatchesAWSsPutObjectExample(t *testing.T) {
+	req, _ := http.NewRequest(http.MethodPut, "http://examplebucket.s3.amazonaws.com/test$file.text", strings.NewReader("Welcome to Amazon S3."))
+	req.Header.Set("Date", "Fri, 24 May 2013 00:00:00 GMT")
+	req.Header.Set("X-Amz-Storage-Class", "REDUCED_REDUNDANCY")
+	payload := hexSHA256([]byte("Welcome to Amazon S3."))
+	if payload != "44ce7dd67c959e0d3524ffac1771dfbba87d2b6b4b4e99e42034a8b803f8b072" {
+		t.Fatalf("payload hash = %s", payload)
+	}
+	sign(req, payload, awsAccessKey, awsSecretKey, "us-east-1", awsTime)
+	want := awsCred + ", SignedHeaders=date;host;x-amz-content-sha256;x-amz-date;x-amz-storage-class, Signature=98ad721746da40c64f1a55b78f14c238d841ea1380cd77a1b5971af0ece108bd"
+	if got := req.Header.Get("Authorization"); got != want {
+		t.Errorf("Authorization =\n%s\nwant\n%s", got, want)
+	}
+}
+
+func TestSignMatchesAWSsListObjectsExample(t *testing.T) {
+	req, _ := http.NewRequest(http.MethodGet, "http://examplebucket.s3.amazonaws.com/?max-keys=2&prefix=J", nil)
+	sign(req, emptySHA256, awsAccessKey, awsSecretKey, "us-east-1", awsTime)
+	want := awsCred + ", SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=34b48302e7b5fa45bde8084f4b7868a86f0a534bc59db6670ed5711ef69dc6f7"
+	if got := req.Header.Get("Authorization"); got != want {
+		t.Errorf("Authorization =\n%s\nwant\n%s", got, want)
+	}
+}
+
+func TestUriEncodeKeepsOnlyUnreservedCharacters(t *testing.T) {
+	for in, want := range map[string]string{
+		"01 - Song.flac": "01%20-%20Song.flac",
+		"a+b=c&d":        "a%2Bb%3Dc%26d",
+		"Björk~_.":       "Bj%C3%B6rk~_.",
+		"a/b":            "a%2Fb",
+	} {
+		if got := uriEncode(in); got != want {
+			t.Errorf("uriEncode(%q) = %q, want %q", in, got, want)
+		}
+	}
+}
+
+func testBucket(url string) Bucket {
+	return Bucket{Endpoint: url, Name: "minstrel", AccessKey: "ak", SecretKey: "sk",
+		now: func() time.Time { return awsTime }}
+}
+
+func TestPutSendsTheBodyToItsPathStyleKey(t *testing.T) {
+	var gotPath, gotSHA, gotBody, gotAuth string
+	var gotLen int64
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		gotPath, gotSHA, gotAuth, gotLen = r.URL.EscapedPath(), r.Header.Get("X-Amz-Content-Sha256"), r.Header.Get("Authorization"), r.ContentLength
+		b, _ := io.ReadAll(r.Body)
+		gotBody = string(b)
+	}))
+	defer srv.Close()
+
+	body := "flac bytes"
+	sum := hexSHA256([]byte(body))
+	if err := testBucket(srv.URL).Put(context.Background(), "music/A B/01 + x.flac", strings.NewReader(body), int64(len(body)), sum); err != nil {
+		t.Fatal(err)
+	}
+	if gotPath != "/minstrel/music/A%20B/01%20%2B%20x.flac" {
+		t.Errorf("path = %s", gotPath)
+	}
+	if gotSHA != sum || gotBody != body || gotLen != int64(len(body)) {
+		t.Errorf("sha %s body %q length %d", gotSHA, gotBody, gotLen)
+	}
+	if !strings.HasPrefix(gotAuth, "AWS4-HMAC-SHA256 Credential=ak/20130524/us-east-1/s3/aws4_request") {
+		t.Errorf("Authorization = %s", gotAuth)
+	}
+}
+
+func TestListFollowsContinuationTokens(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		q := r.URL.Query()
+		if r.URL.Path != "/minstrel" || q.Get("list-type") != "2" || q.Get("prefix") != "database/" {
+			http.Error(w, "unexpected "+r.URL.String(), http.StatusBadRequest)
+			return
+		}
+		if q.Get("continuation-token") == "" {
+			fmt.Fprint(w, `<ListBucketResult><IsTruncated>true</IsTruncated><NextContinuationToken>next</NextContinuationToken>
+				<Contents><Key>database/minstrel-20261101T030000Z.tar</Key><Size>10</Size><LastModified>2026-11-01T03:00:05.000Z</LastModified></Contents>
+				<Contents><Key>database/minstrel-20261102T030000Z.tar</Key><Size>11</Size><LastModified>2026-11-02T03:00:05.000Z</LastModified></Contents>
+				</ListBucketResult>`)
+			return
+		}
+		fmt.Fprint(w, `<ListBucketResult><IsTruncated>false</IsTruncated>
+			<Contents><Key>database/minstrel-20261103T030000Z.tar</Key><Size>12</Size><LastModified>2026-11-03T03:00:05.000Z</LastModified></Contents>
+			</ListBucketResult>`)
+	}))
+	defer srv.Close()
+
+	objs, err := testBucket(srv.URL).List(context.Background(), "database/", 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(objs) != 3 || objs[2].Key != "database/minstrel-20261103T030000Z.tar" || objs[1].Size != 11 ||
+		!objs[0].LastModified.Equal(time.Date(2026, 11, 1, 3, 0, 5, 0, time.UTC)) {
+		t.Errorf("objects = %+v", objs)
+	}
+}
+
+func TestARefusalSaysWhatIsWrong(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusForbidden)
+		fmt.Fprint(w, `<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match</Message></Error>`)
+	}))
+	defer srv.Close()
+
+	_, err := testBucket(srv.URL).List(context.Background(), "", 1)
+	if !errors.Is(err, ErrS3) || !strings.Contains(err.Error(), "HTTP 403 SignatureDoesNotMatch: the secret key is wrong") {
+		t.Errorf("err = %v", err)
+	}
+}
+
+func TestAnUnreachableOrMalformedEndpointIsAnS3Error(t *testing.T) {
+	srv := httptest.NewServer(http.NotFoundHandler())
+	srv.Close()
+	for name, endpoint := range map[string]string{"closed": srv.URL, "not http": "ftp://archives", "empty": ""} {
+		if _, err := testBucket(endpoint).List(context.Background(), "", 1); !errors.Is(err, ErrS3) {
+			t.Errorf("%s: err = %v, want ErrS3", name, err)
+		}
+	}
+}

internal/backup/sigv4.go

@@ -0,0 +1,119 @@
+package backup
+
+import (
+	"crypto/hmac"
+	"crypto/sha256"
+	"encoding/hex"
+	"net/http"
+	"net/url"
+	"slices"
+	"strings"
+	"time"
+)
+
+// emptySHA256 is the payload hash of a request with no body.
+const emptySHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+
+// sign adds AWS Signature Version 4 headers for S3 to req: X-Amz-Date,
+// X-Amz-Content-Sha256 and Authorization. Every header already on the
+// request is signed, along with the host.
+func sign(req *http.Request, payloadSHA256, accessKey, secretKey, region string, at time.Time) {
+	at = at.UTC()
+	stamp := at.Format("20060102T150405Z")
+	date := at.Format("20060102")
+	req.Header.Set("X-Amz-Date", stamp)
+	req.Header.Set("X-Amz-Content-Sha256", payloadSHA256)
+
+	host := req.Host
+	if host == "" {
+		host = req.URL.Host
+	}
+	headers := map[string]string{"host": host}
+	for name, values := range req.Header {
+		headers[strings.ToLower(name)] = strings.TrimSpace(strings.Join(values, ","))
+	}
+	names := make([]string, 0, len(headers))
+	for name := range headers {
+		names = append(names, name)
+	}
+	slices.Sort(names)
+	var canonHeaders strings.Builder
+	for _, name := range names {
+		canonHeaders.WriteString(name + ":" + headers[name] + "\n")
+	}
+	signed := strings.Join(names, ";")
+
+	canonical := strings.Join([]string{
+		req.Method,
+		canonicalPath(req.URL),
+		canonicalQuery(req.URL.Query()),
+		canonHeaders.String(),
+		signed,
+		payloadSHA256,
+	}, "\n")
+	scope := date + "/" + region + "/s3/aws4_request"
+	toSign := "AWS4-HMAC-SHA256\n" + stamp + "\n" + scope + "\n" + hexSHA256([]byte(canonical))
+
+	key := hmacSHA256([]byte("AWS4"+secretKey), date)
+	key = hmacSHA256(key, region)
+	key = hmacSHA256(key, "s3")
+	key = hmacSHA256(key, "aws4_request")
+	signature := hex.EncodeToString(hmacSHA256(key, toSign))
+
+	req.Header.Set("Authorization", "AWS4-HMAC-SHA256 Credential="+accessKey+"/"+scope+
+		", SignedHeaders="+signed+", Signature="+signature)
+}
+
+// canonicalPath is the request path with each segment URI-encoded once.
+func canonicalPath(u *url.URL) string {
+	p := u.EscapedPath()
+	if p == "" {
+		return "/"
+	}
+	if unescaped, err := url.PathUnescape(p); err == nil {
+		p = unescaped
+	}
+	segments := strings.Split(p, "/")
+	for i, s := range segments {
+		segments[i] = uriEncode(s)
+	}
+	return strings.Join(segments, "/")
+}
+
+// canonicalQuery is the query sorted by name, names and values URI-encoded.
+func canonicalQuery(q url.Values) string {
+	pairs := make([]string, 0, len(q))
+	for name, values := range q {
+		for _, v := range values {
+			pairs = append(pairs, uriEncode(name)+"="+uriEncode(v))
+		}
+	}
+	slices.Sort(pairs)
+	return strings.Join(pairs, "&")
+}
+
+// uriEncode percent-encodes everything but the unreserved characters, as
+// SigV4 requires; url.QueryEscape differs on space and '~'.
+func uriEncode(s string) string {
+	var b strings.Builder
+	for i := 0; i < len(s); i++ {
+		c := s[i]
+		if c >= 'A' && c <= 'Z' || c >= 'a' && c <= 'z' || c >= '0' && c <= '9' || c == '-' || c == '_' || c == '.' || c == '~' {
+			b.WriteByte(c)
+			continue
+		}
+		b.WriteString("%" + strings.ToUpper(hex.EncodeToString([]byte{c})))
+	}
+	return b.String()
+}
+
+func hmacSHA256(key []byte, msg string) []byte {
+	m := hmac.New(sha256.New, key)
+	m.Write([]byte(msg))
+	return m.Sum(nil)
+}
+
+func hexSHA256(b []byte) string {
+	sum := sha256.Sum256(b)
+	return hex.EncodeToString(sum[:])
+}