From b6ae20586291800f410b97890fdda1ba51638974 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Mon, 2 Nov 2026 13:51:02 -0500 Subject: [PATCH] feat: Minstrel can put and list objects in a path-style S3 bucket, signed with SigV4 and no SDK (#6771) Bucket.Put stores a body under a key, signed over its SHA-256; Bucket.List pages through ListObjectsV2. A backup only adds and lists, so an append-only key is enough. Keys are encoded once, exactly as signed, so music paths with spaces, plus signs and accents reach the bucket intact. A refusal names S3's code in words ("the secret key is wrong"). The signer is pinned to AWS's published GET Object, PUT Object and ListObjects examples. Co-Authored-By: Claude Opus 5.5 --- internal/backup/s3.go | 201 +++++++++++++++++++++++++++++++++++++ internal/backup/s3_test.go | 154 ++++++++++++++++++++++++++++ internal/backup/sigv4.go | 119 ++++++++++++++++++++++ 3 files changed, 474 insertions(+) create mode 100644 internal/backup/s3.go create mode 100644 internal/backup/s3_test.go create mode 100644 internal/backup/sigv4.go diff --git a/internal/backup/s3.go b/internal/backup/s3.go new file mode 100644 index 00000000..d69ad11b --- /dev/null +++ b/internal/backup/s3.go @@ -0,0 +1,201 @@ +package backup + +import ( + "context" + "encoding/xml" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "strings" + "time" +) + +// ErrS3 is a request the bucket refused or never answered. +var ErrS3 = errors.New("backup: the S3 bucket refused the request") + +// Bucket is a path-style S3 bucket: an endpoint such as Fabled Archives' +// backup port or MinIO, a bucket name, and a key pair. A backup only ever +// adds objects and lists them, so an append-only key is enough. +type Bucket struct { + Endpoint string + Name string + AccessKey string + SecretKey string + // Region defaults to us-east-1, which S3-compatible servers accept. + Region string + Client *http.Client + // now is the signing clock; tests pin it. + now func() time.Time +} + +// Object is one stored object. +type Object struct { + Key string + Size int64 + LastModified time.Time +} + +func (b Bucket) region() string { + if b.Region == "" { + return "us-east-1" + } + return b.Region +} + +func (b Bucket) client() *http.Client { + if b.Client != nil { + return b.Client + } + return http.DefaultClient +} + +func (b Bucket) clock() time.Time { + if b.now != nil { + return b.now() + } + return time.Now() +} + +// objectURL is the path-style URL of key, encoded exactly as it is signed. +func (b Bucket) objectURL(key string) (*url.URL, error) { + u, err := url.Parse(strings.TrimRight(b.Endpoint, "/")) + if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" { + return nil, fmt.Errorf("%w: the endpoint %q is not an http(s) address", ErrS3, b.Endpoint) + } + if b.Name == "" { + return nil, fmt.Errorf("%w: no bucket name", ErrS3) + } + raw := "/" + uriEncode(b.Name) + if key != "" { + for s := range strings.SplitSeq(key, "/") { + raw += "/" + uriEncode(s) + } + } + u.RawPath = u.EscapedPath() + raw + u.Path, _ = url.PathUnescape(u.RawPath) + return u, nil +} + +func (b Bucket) do(req *http.Request, payloadSHA256 string) (*http.Response, error) { + sign(req, payloadSHA256, b.AccessKey, b.SecretKey, b.region(), b.clock()) + resp, err := b.client().Do(req) + if err != nil { + return nil, fmt.Errorf("%w: %s is unreachable: %v", ErrS3, req.URL.Host, err) + } + if resp.StatusCode/100 != 2 { + defer resp.Body.Close() + return nil, refusal(resp) + } + return resp, nil +} + +// Put stores body under key. size is the body's length and payloadSHA256 +// its hex SHA-256, which the request is signed over. +func (b Bucket) Put(ctx context.Context, key string, body io.Reader, size int64, payloadSHA256 string) error { + u, err := b.objectURL(key) + if err != nil { + return err + } + req, err := http.NewRequestWithContext(ctx, http.MethodPut, u.String(), body) + if err != nil { + return err + } + req.URL = u + req.ContentLength = size + if size == 0 { + req.Body = http.NoBody + } + resp, err := b.do(req, payloadSHA256) + if err != nil { + return err + } + drain(resp) + return nil +} + +// List returns up to limit objects whose keys start with prefix, in key +// order. +func (b Bucket) List(ctx context.Context, prefix string, limit int) ([]Object, error) { + if limit < 1 { + limit = 1000 + } + var out []Object + token := "" + for { + u, err := b.objectURL("") + if err != nil { + return nil, err + } + q := url.Values{"list-type": {"2"}, "prefix": {prefix}} + if page := limit - len(out); page < 1000 { + q.Set("max-keys", strconv.Itoa(page)) + } + if token != "" { + q.Set("continuation-token", token) + } + u.RawQuery = canonicalQuery(q) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + return nil, err + } + req.URL = u + resp, err := b.do(req, emptySHA256) + if err != nil { + return nil, err + } + var page struct { + Contents []struct { + Key string `xml:"Key"` + Size int64 `xml:"Size"` + LastModified string `xml:"LastModified"` + } `xml:"Contents"` + IsTruncated bool `xml:"IsTruncated"` + NextContinuationToken string `xml:"NextContinuationToken"` + } + err = xml.NewDecoder(io.LimitReader(resp.Body, 16<<20)).Decode(&page) + drain(resp) + if err != nil { + return nil, fmt.Errorf("%w: the listing is not S3 XML: %v", ErrS3, err) + } + for _, c := range page.Contents { + modified, _ := time.Parse(time.RFC3339, c.LastModified) + out = append(out, Object{Key: c.Key, Size: c.Size, LastModified: modified}) + } + if !page.IsTruncated || page.NextContinuationToken == "" || len(out) >= limit { + return out, nil + } + token = page.NextContinuationToken + } +} + +// refusal turns an S3 error response into words an admin can act on. +func refusal(resp *http.Response) error { + var e struct { + Code string `xml:"Code"` + Message string `xml:"Message"` + } + _ = xml.NewDecoder(io.LimitReader(resp.Body, 64<<10)).Decode(&e) + hint := map[string]string{ + "SignatureDoesNotMatch": "the secret key is wrong", + "InvalidAccessKeyId": "the access key is not known to this server", + "AccessDenied": "the key may not do this in this bucket", + "NoSuchBucket": "there is no bucket by that name", + "SlowDown": "the server asks to slow down", + }[e.Code] + switch { + case hint != "": + return fmt.Errorf("%w: HTTP %d %s: %s", ErrS3, resp.StatusCode, e.Code, hint) + case e.Code != "": + return fmt.Errorf("%w: HTTP %d %s: %s", ErrS3, resp.StatusCode, e.Code, e.Message) + default: + return fmt.Errorf("%w: HTTP %d", ErrS3, resp.StatusCode) + } +} + +func drain(resp *http.Response) { + _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 1<<20)) + _ = resp.Body.Close() +} diff --git a/internal/backup/s3_test.go b/internal/backup/s3_test.go new file mode 100644 index 00000000..543e225d --- /dev/null +++ b/internal/backup/s3_test.go @@ -0,0 +1,154 @@ +package backup + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +// AWS's published examples ("Examples: Signature Calculations in AWS +// Signature Version 4", Amazon S3 API reference). +const ( + awsAccessKey = "AKIAIOSFODNN7EXAMPLE" + awsSecretKey = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + awsCred = "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request" +) + +var awsTime = time.Date(2013, 5, 24, 0, 0, 0, 0, time.UTC) + +func TestSignMatchesAWSsGetObjectExample(t *testing.T) { + req, _ := http.NewRequest(http.MethodGet, "http://examplebucket.s3.amazonaws.com/test.txt", nil) + req.Header.Set("Range", "bytes=0-9") + sign(req, emptySHA256, awsAccessKey, awsSecretKey, "us-east-1", awsTime) + want := awsCred + ", SignedHeaders=host;range;x-amz-content-sha256;x-amz-date, Signature=f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41" + if got := req.Header.Get("Authorization"); got != want { + t.Errorf("Authorization =\n%s\nwant\n%s", got, want) + } +} + +func TestSignMatchesAWSsPutObjectExample(t *testing.T) { + req, _ := http.NewRequest(http.MethodPut, "http://examplebucket.s3.amazonaws.com/test$file.text", strings.NewReader("Welcome to Amazon S3.")) + req.Header.Set("Date", "Fri, 24 May 2013 00:00:00 GMT") + req.Header.Set("X-Amz-Storage-Class", "REDUCED_REDUNDANCY") + payload := hexSHA256([]byte("Welcome to Amazon S3.")) + if payload != "44ce7dd67c959e0d3524ffac1771dfbba87d2b6b4b4e99e42034a8b803f8b072" { + t.Fatalf("payload hash = %s", payload) + } + sign(req, payload, awsAccessKey, awsSecretKey, "us-east-1", awsTime) + want := awsCred + ", SignedHeaders=date;host;x-amz-content-sha256;x-amz-date;x-amz-storage-class, Signature=98ad721746da40c64f1a55b78f14c238d841ea1380cd77a1b5971af0ece108bd" + if got := req.Header.Get("Authorization"); got != want { + t.Errorf("Authorization =\n%s\nwant\n%s", got, want) + } +} + +func TestSignMatchesAWSsListObjectsExample(t *testing.T) { + req, _ := http.NewRequest(http.MethodGet, "http://examplebucket.s3.amazonaws.com/?max-keys=2&prefix=J", nil) + sign(req, emptySHA256, awsAccessKey, awsSecretKey, "us-east-1", awsTime) + want := awsCred + ", SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=34b48302e7b5fa45bde8084f4b7868a86f0a534bc59db6670ed5711ef69dc6f7" + if got := req.Header.Get("Authorization"); got != want { + t.Errorf("Authorization =\n%s\nwant\n%s", got, want) + } +} + +func TestUriEncodeKeepsOnlyUnreservedCharacters(t *testing.T) { + for in, want := range map[string]string{ + "01 - Song.flac": "01%20-%20Song.flac", + "a+b=c&d": "a%2Bb%3Dc%26d", + "Björk~_.": "Bj%C3%B6rk~_.", + "a/b": "a%2Fb", + } { + if got := uriEncode(in); got != want { + t.Errorf("uriEncode(%q) = %q, want %q", in, got, want) + } + } +} + +func testBucket(url string) Bucket { + return Bucket{Endpoint: url, Name: "minstrel", AccessKey: "ak", SecretKey: "sk", + now: func() time.Time { return awsTime }} +} + +func TestPutSendsTheBodyToItsPathStyleKey(t *testing.T) { + var gotPath, gotSHA, gotBody, gotAuth string + var gotLen int64 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath, gotSHA, gotAuth, gotLen = r.URL.EscapedPath(), r.Header.Get("X-Amz-Content-Sha256"), r.Header.Get("Authorization"), r.ContentLength + b, _ := io.ReadAll(r.Body) + gotBody = string(b) + })) + defer srv.Close() + + body := "flac bytes" + sum := hexSHA256([]byte(body)) + if err := testBucket(srv.URL).Put(context.Background(), "music/A B/01 + x.flac", strings.NewReader(body), int64(len(body)), sum); err != nil { + t.Fatal(err) + } + if gotPath != "/minstrel/music/A%20B/01%20%2B%20x.flac" { + t.Errorf("path = %s", gotPath) + } + if gotSHA != sum || gotBody != body || gotLen != int64(len(body)) { + t.Errorf("sha %s body %q length %d", gotSHA, gotBody, gotLen) + } + if !strings.HasPrefix(gotAuth, "AWS4-HMAC-SHA256 Credential=ak/20130524/us-east-1/s3/aws4_request") { + t.Errorf("Authorization = %s", gotAuth) + } +} + +func TestListFollowsContinuationTokens(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + if r.URL.Path != "/minstrel" || q.Get("list-type") != "2" || q.Get("prefix") != "database/" { + http.Error(w, "unexpected "+r.URL.String(), http.StatusBadRequest) + return + } + if q.Get("continuation-token") == "" { + fmt.Fprint(w, `truenext + database/minstrel-20261101T030000Z.tar102026-11-01T03:00:05.000Z + database/minstrel-20261102T030000Z.tar112026-11-02T03:00:05.000Z + `) + return + } + fmt.Fprint(w, `false + database/minstrel-20261103T030000Z.tar122026-11-03T03:00:05.000Z + `) + })) + defer srv.Close() + + objs, err := testBucket(srv.URL).List(context.Background(), "database/", 0) + if err != nil { + t.Fatal(err) + } + if len(objs) != 3 || objs[2].Key != "database/minstrel-20261103T030000Z.tar" || objs[1].Size != 11 || + !objs[0].LastModified.Equal(time.Date(2026, 11, 1, 3, 0, 5, 0, time.UTC)) { + t.Errorf("objects = %+v", objs) + } +} + +func TestARefusalSaysWhatIsWrong(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusForbidden) + fmt.Fprint(w, `SignatureDoesNotMatchThe request signature we calculated does not match`) + })) + defer srv.Close() + + _, err := testBucket(srv.URL).List(context.Background(), "", 1) + if !errors.Is(err, ErrS3) || !strings.Contains(err.Error(), "HTTP 403 SignatureDoesNotMatch: the secret key is wrong") { + t.Errorf("err = %v", err) + } +} + +func TestAnUnreachableOrMalformedEndpointIsAnS3Error(t *testing.T) { + srv := httptest.NewServer(http.NotFoundHandler()) + srv.Close() + for name, endpoint := range map[string]string{"closed": srv.URL, "not http": "ftp://archives", "empty": ""} { + if _, err := testBucket(endpoint).List(context.Background(), "", 1); !errors.Is(err, ErrS3) { + t.Errorf("%s: err = %v, want ErrS3", name, err) + } + } +} diff --git a/internal/backup/sigv4.go b/internal/backup/sigv4.go new file mode 100644 index 00000000..ee676819 --- /dev/null +++ b/internal/backup/sigv4.go @@ -0,0 +1,119 @@ +package backup + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "net/http" + "net/url" + "slices" + "strings" + "time" +) + +// emptySHA256 is the payload hash of a request with no body. +const emptySHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + +// sign adds AWS Signature Version 4 headers for S3 to req: X-Amz-Date, +// X-Amz-Content-Sha256 and Authorization. Every header already on the +// request is signed, along with the host. +func sign(req *http.Request, payloadSHA256, accessKey, secretKey, region string, at time.Time) { + at = at.UTC() + stamp := at.Format("20060102T150405Z") + date := at.Format("20060102") + req.Header.Set("X-Amz-Date", stamp) + req.Header.Set("X-Amz-Content-Sha256", payloadSHA256) + + host := req.Host + if host == "" { + host = req.URL.Host + } + headers := map[string]string{"host": host} + for name, values := range req.Header { + headers[strings.ToLower(name)] = strings.TrimSpace(strings.Join(values, ",")) + } + names := make([]string, 0, len(headers)) + for name := range headers { + names = append(names, name) + } + slices.Sort(names) + var canonHeaders strings.Builder + for _, name := range names { + canonHeaders.WriteString(name + ":" + headers[name] + "\n") + } + signed := strings.Join(names, ";") + + canonical := strings.Join([]string{ + req.Method, + canonicalPath(req.URL), + canonicalQuery(req.URL.Query()), + canonHeaders.String(), + signed, + payloadSHA256, + }, "\n") + scope := date + "/" + region + "/s3/aws4_request" + toSign := "AWS4-HMAC-SHA256\n" + stamp + "\n" + scope + "\n" + hexSHA256([]byte(canonical)) + + key := hmacSHA256([]byte("AWS4"+secretKey), date) + key = hmacSHA256(key, region) + key = hmacSHA256(key, "s3") + key = hmacSHA256(key, "aws4_request") + signature := hex.EncodeToString(hmacSHA256(key, toSign)) + + req.Header.Set("Authorization", "AWS4-HMAC-SHA256 Credential="+accessKey+"/"+scope+ + ", SignedHeaders="+signed+", Signature="+signature) +} + +// canonicalPath is the request path with each segment URI-encoded once. +func canonicalPath(u *url.URL) string { + p := u.EscapedPath() + if p == "" { + return "/" + } + if unescaped, err := url.PathUnescape(p); err == nil { + p = unescaped + } + segments := strings.Split(p, "/") + for i, s := range segments { + segments[i] = uriEncode(s) + } + return strings.Join(segments, "/") +} + +// canonicalQuery is the query sorted by name, names and values URI-encoded. +func canonicalQuery(q url.Values) string { + pairs := make([]string, 0, len(q)) + for name, values := range q { + for _, v := range values { + pairs = append(pairs, uriEncode(name)+"="+uriEncode(v)) + } + } + slices.Sort(pairs) + return strings.Join(pairs, "&") +} + +// uriEncode percent-encodes everything but the unreserved characters, as +// SigV4 requires; url.QueryEscape differs on space and '~'. +func uriEncode(s string) string { + var b strings.Builder + for i := 0; i < len(s); i++ { + c := s[i] + if c >= 'A' && c <= 'Z' || c >= 'a' && c <= 'z' || c >= '0' && c <= '9' || c == '-' || c == '_' || c == '.' || c == '~' { + b.WriteByte(c) + continue + } + b.WriteString("%" + strings.ToUpper(hex.EncodeToString([]byte{c}))) + } + return b.String() +} + +func hmacSHA256(key []byte, msg string) []byte { + m := hmac.New(sha256.New, key) + m.Write([]byte(msg)) + return m.Sum(nil) +} + +func hexSHA256(b []byte) string { + sum := sha256.Sum256(b) + return hex.EncodeToString(sum[:]) +}