feat: Minstrel can put and list objects in a path-style S3 bucket, signed with SigV4 and no SDK (#6771)

Bucket.Put stores a body under a key, signed over its SHA-256; Bucket.List
pages through ListObjectsV2. A backup only adds and lists, so an
append-only key is enough. Keys are encoded once, exactly as signed, so
music paths with spaces, plus signs and accents reach the bucket intact.
A refusal names S3's code in words ("the secret key is wrong").

The signer is pinned to AWS's published GET Object, PUT Object and
ListObjects examples.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/backup/s3_test.go

@@ -0,0 +1,154 @@
+package backup
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+)
+
+// AWS's published examples ("Examples: Signature Calculations in AWS
+// Signature Version 4", Amazon S3 API reference).
+const (
+	awsAccessKey = "AKIAIOSFODNN7EXAMPLE"
+	awsSecretKey = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
+	awsCred      = "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request"
+)
+
+var awsTime = time.Date(2013, 5, 24, 0, 0, 0, 0, time.UTC)
+
+func TestSignMatchesAWSsGetObjectExample(t *testing.T) {
+	req, _ := http.NewRequest(http.MethodGet, "http://examplebucket.s3.amazonaws.com/test.txt", nil)
+	req.Header.Set("Range", "bytes=0-9")
+	sign(req, emptySHA256, awsAccessKey, awsSecretKey, "us-east-1", awsTime)
+	want := awsCred + ", SignedHeaders=host;range;x-amz-content-sha256;x-amz-date, Signature=f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41"
+	if got := req.Header.Get("Authorization"); got != want {
+		t.Errorf("Authorization =\n%s\nwant\n%s", got, want)
+	}
+}
+
+func TestSignMatchesAWSsPutObjectExample(t *testing.T) {
+	req, _ := http.NewRequest(http.MethodPut, "http://examplebucket.s3.amazonaws.com/test$file.text", strings.NewReader("Welcome to Amazon S3."))
+	req.Header.Set("Date", "Fri, 24 May 2013 00:00:00 GMT")
+	req.Header.Set("X-Amz-Storage-Class", "REDUCED_REDUNDANCY")
+	payload := hexSHA256([]byte("Welcome to Amazon S3."))
+	if payload != "44ce7dd67c959e0d3524ffac1771dfbba87d2b6b4b4e99e42034a8b803f8b072" {
+		t.Fatalf("payload hash = %s", payload)
+	}
+	sign(req, payload, awsAccessKey, awsSecretKey, "us-east-1", awsTime)
+	want := awsCred + ", SignedHeaders=date;host;x-amz-content-sha256;x-amz-date;x-amz-storage-class, Signature=98ad721746da40c64f1a55b78f14c238d841ea1380cd77a1b5971af0ece108bd"
+	if got := req.Header.Get("Authorization"); got != want {
+		t.Errorf("Authorization =\n%s\nwant\n%s", got, want)
+	}
+}
+
+func TestSignMatchesAWSsListObjectsExample(t *testing.T) {
+	req, _ := http.NewRequest(http.MethodGet, "http://examplebucket.s3.amazonaws.com/?max-keys=2&prefix=J", nil)
+	sign(req, emptySHA256, awsAccessKey, awsSecretKey, "us-east-1", awsTime)
+	want := awsCred + ", SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=34b48302e7b5fa45bde8084f4b7868a86f0a534bc59db6670ed5711ef69dc6f7"
+	if got := req.Header.Get("Authorization"); got != want {
+		t.Errorf("Authorization =\n%s\nwant\n%s", got, want)
+	}
+}
+
+func TestUriEncodeKeepsOnlyUnreservedCharacters(t *testing.T) {
+	for in, want := range map[string]string{
+		"01 - Song.flac": "01%20-%20Song.flac",
+		"a+b=c&d":        "a%2Bb%3Dc%26d",
+		"Björk~_.":       "Bj%C3%B6rk~_.",
+		"a/b":            "a%2Fb",
+	} {
+		if got := uriEncode(in); got != want {
+			t.Errorf("uriEncode(%q) = %q, want %q", in, got, want)
+		}
+	}
+}
+
+func testBucket(url string) Bucket {
+	return Bucket{Endpoint: url, Name: "minstrel", AccessKey: "ak", SecretKey: "sk",
+		now: func() time.Time { return awsTime }}
+}
+
+func TestPutSendsTheBodyToItsPathStyleKey(t *testing.T) {
+	var gotPath, gotSHA, gotBody, gotAuth string
+	var gotLen int64
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		gotPath, gotSHA, gotAuth, gotLen = r.URL.EscapedPath(), r.Header.Get("X-Amz-Content-Sha256"), r.Header.Get("Authorization"), r.ContentLength
+		b, _ := io.ReadAll(r.Body)
+		gotBody = string(b)
+	}))
+	defer srv.Close()
+
+	body := "flac bytes"
+	sum := hexSHA256([]byte(body))
+	if err := testBucket(srv.URL).Put(context.Background(), "music/A B/01 + x.flac", strings.NewReader(body), int64(len(body)), sum); err != nil {
+		t.Fatal(err)
+	}
+	if gotPath != "/minstrel/music/A%20B/01%20%2B%20x.flac" {
+		t.Errorf("path = %s", gotPath)
+	}
+	if gotSHA != sum || gotBody != body || gotLen != int64(len(body)) {
+		t.Errorf("sha %s body %q length %d", gotSHA, gotBody, gotLen)
+	}
+	if !strings.HasPrefix(gotAuth, "AWS4-HMAC-SHA256 Credential=ak/20130524/us-east-1/s3/aws4_request") {
+		t.Errorf("Authorization = %s", gotAuth)
+	}
+}
+
+func TestListFollowsContinuationTokens(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		q := r.URL.Query()
+		if r.URL.Path != "/minstrel" || q.Get("list-type") != "2" || q.Get("prefix") != "database/" {
+			http.Error(w, "unexpected "+r.URL.String(), http.StatusBadRequest)
+			return
+		}
+		if q.Get("continuation-token") == "" {
+			fmt.Fprint(w, `<ListBucketResult><IsTruncated>true</IsTruncated><NextContinuationToken>next</NextContinuationToken>
+				<Contents><Key>database/minstrel-20261101T030000Z.tar</Key><Size>10</Size><LastModified>2026-11-01T03:00:05.000Z</LastModified></Contents>
+				<Contents><Key>database/minstrel-20261102T030000Z.tar</Key><Size>11</Size><LastModified>2026-11-02T03:00:05.000Z</LastModified></Contents>
+				</ListBucketResult>`)
+			return
+		}
+		fmt.Fprint(w, `<ListBucketResult><IsTruncated>false</IsTruncated>
+			<Contents><Key>database/minstrel-20261103T030000Z.tar</Key><Size>12</Size><LastModified>2026-11-03T03:00:05.000Z</LastModified></Contents>
+			</ListBucketResult>`)
+	}))
+	defer srv.Close()
+
+	objs, err := testBucket(srv.URL).List(context.Background(), "database/", 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(objs) != 3 || objs[2].Key != "database/minstrel-20261103T030000Z.tar" || objs[1].Size != 11 ||
+		!objs[0].LastModified.Equal(time.Date(2026, 11, 1, 3, 0, 5, 0, time.UTC)) {
+		t.Errorf("objects = %+v", objs)
+	}
+}
+
+func TestARefusalSaysWhatIsWrong(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusForbidden)
+		fmt.Fprint(w, `<Error><Code>SignatureDoesNotMatch</Code><Message>The request signature we calculated does not match</Message></Error>`)
+	}))
+	defer srv.Close()
+
+	_, err := testBucket(srv.URL).List(context.Background(), "", 1)
+	if !errors.Is(err, ErrS3) || !strings.Contains(err.Error(), "HTTP 403 SignatureDoesNotMatch: the secret key is wrong") {
+		t.Errorf("err = %v", err)
+	}
+}
+
+func TestAnUnreachableOrMalformedEndpointIsAnS3Error(t *testing.T) {
+	srv := httptest.NewServer(http.NotFoundHandler())
+	srv.Close()
+	for name, endpoint := range map[string]string{"closed": srv.URL, "not http": "ftp://archives", "empty": ""} {
+		if _, err := testBucket(endpoint).List(context.Background(), "", 1); !errors.Is(err, ErrS3) {
+			t.Errorf("%s: err = %v, want ErrS3", name, err)
+		}
+	}
+}