From da1de165b8a8d7440b7d6605933f6943e823b384 Mon Sep 17 00:00:00 2001 From: Kenta Miyachi Date: Thu, 20 Jun 2019 15:05:45 +0900 Subject: [PATCH 1/2] using offical signer --- s3.go | 46 +++++++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/s3.go b/s3.go index a32bdb9..67ab401 100644 --- a/s3.go +++ b/s3.go @@ -2,9 +2,9 @@ package s3 import ( "bytes" - "crypto/hmac" + // "crypto/hmac" "crypto/md5" - "crypto/sha1" + // "crypto/sha1" "encoding/base64" "encoding/xml" "fmt" @@ -17,6 +17,11 @@ import ( "sort" "strings" "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + // "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/aws/signer/v4" ) // S3 provides a wrapper around your S3 credentials. It carries no other internal state @@ -39,14 +44,16 @@ func NewS3(bucket, accessId, secret string) *S3 { } func (s3 *S3) signRequest(req *http.Request) { - amzHeaders := "" + singer := v4.Signer{ + Credentials: credentials.NewStaticCredentials(s3.accessId, s3.secret, ""), + } resourceUrl, _ := url.Parse("/" + s3.bucket + req.URL.Path) resource := resourceUrl.String() - /* Ugh, AWS requires us to order the parameters in a specific ordering for - * signing. Makes sense, but is annoying because a map does not have a defined - * ordering (and basically returns elements in a random order) -- so we have - * to sort by hand */ + // /* Ugh, AWS requires us to order the parameters in a specific ordering for + // * signing. Makes sense, but is annoying because a map does not have a defined + // * ordering (and basically returns elements in a random order) -- so we have + // * to sort by hand */ query := req.URL.Query() if len(query) > 0 { keys := []string{} @@ -84,20 +91,17 @@ func (s3 *S3) signRequest(req *http.Request) { req.Header.Set("Date", time.Now().Format(time.RFC1123)) } - authStr := strings.Join([]string{ - strings.TrimSpace(req.Method), - req.Header.Get("Content-MD5"), - req.Header.Get("Content-Type"), - req.Header.Get("Date"), - amzHeaders + resource, - }, "\n") - - h := hmac.New(sha1.New, []byte(s3.secret)) - h.Write([]byte(authStr)) - - h64 := base64.StdEncoding.EncodeToString(h.Sum(nil)) - auth := "AWS" + " " + s3.accessId + ":" + h64 - req.Header.Set("Authorization", auth) + var seeker io.ReadSeeker + if sr, ok := req.Body.(io.ReadSeeker); ok { + seeker = sr + } else { + seeker = aws.ReadSeekCloser(req.Body) + } + signedHeaders, err := singer.Sign(req, seeker, "s3", "ap-northeast-1", time.Now()) + if err != nil { + fmt.Printf("error: %s\n", err) + } + fmt.Printf("Signed Header: %s\n", signedHeaders) } func (s3 *S3) resource(path string, values url.Values) string { From 4d92948cc0c9324b4b2ecd39c533ae68b9bd15bd Mon Sep 17 00:00:00 2001 From: Kenta Miyachi Date: Fri, 21 Jun 2019 14:07:56 +0900 Subject: [PATCH 2/2] adjusting for v4 signature --- multipart.go | 2 -- s3.go | 96 +++++++++++++++++++--------------------------------- 2 files changed, 34 insertions(+), 64 deletions(-) diff --git a/multipart.go b/multipart.go index d7322a7..e5e4ed8 100644 --- a/multipart.go +++ b/multipart.go @@ -58,7 +58,6 @@ func (mp *S3Multipart) AddPart(r io.Reader, size int64, md5sum []byte) error { } req.Header.Set("Content-Length", fmt.Sprintf("%d", size)) - req.Header.Set("Host", req.URL.Host) req.Header.Set("Content-Type", "application/octet-stream") req.ContentLength = size @@ -114,7 +113,6 @@ func (mp *S3Multipart) Complete(contentType string) error { } req.Header.Set("Content-Length", fmt.Sprintf("%d", len(xmlBody))) - req.Header.Set("Host", req.URL.Host) req.Header.Set("Content-Type", contentType) req.ContentLength = int64(len(xmlBody)) diff --git a/s3.go b/s3.go index 67ab401..f95d24f 100644 --- a/s3.go +++ b/s3.go @@ -2,9 +2,7 @@ package s3 import ( "bytes" - // "crypto/hmac" "crypto/md5" - // "crypto/sha1" "encoding/base64" "encoding/xml" "fmt" @@ -14,13 +12,11 @@ import ( "net/http" "net/url" "runtime" - "sort" "strings" "time" - "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/credentials" - // "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/aws/endpoints" "github.com/aws/aws-sdk-go/aws/signer/v4" ) @@ -43,65 +39,32 @@ func NewS3(bucket, accessId, secret string) *S3 { } } -func (s3 *S3) signRequest(req *http.Request) { - singer := v4.Signer{ - Credentials: credentials.NewStaticCredentials(s3.accessId, s3.secret, ""), - } - resourceUrl, _ := url.Parse("/" + s3.bucket + req.URL.Path) - resource := resourceUrl.String() - - // /* Ugh, AWS requires us to order the parameters in a specific ordering for - // * signing. Makes sense, but is annoying because a map does not have a defined - // * ordering (and basically returns elements in a random order) -- so we have - // * to sort by hand */ - query := req.URL.Query() - if len(query) > 0 { - keys := []string{} - - for k := range query { - keys = append(keys, k) - } - - sort.Strings(keys) - - parts := []string{} - - for _, key := range keys { - vals := query[key] - - for _, val := range vals { - if val == "" { - parts = append(parts, url.QueryEscape(key)) - - } else { - part := fmt.Sprintf("%s=%s", url.QueryEscape(key), url.QueryEscape(val)) - parts = append(parts, part) - } - } - } - - req.URL.RawQuery = strings.Join(parts, "&") - } - - if req.URL.RawQuery != "" { - resource += "?" + req.URL.RawQuery +func (s4 *S3) signRequest(req *http.Request) (er error) { + signer := v4.Signer{ + Credentials: credentials.NewStaticCredentials(s4.accessId, s4.secret, ""), + DisableURIPathEscaping: true, } + t := time.Now() if req.Header.Get("Date") == "" { - req.Header.Set("Date", time.Now().Format(time.RFC1123)) + req.Header.Set("Date", t.Format(time.RFC1123)) } var seeker io.ReadSeeker - if sr, ok := req.Body.(io.ReadSeeker); ok { - seeker = sr - } else { - seeker = aws.ReadSeekCloser(req.Body) + if req.Body != nil { + seeker = bytes.NewReader(streamToByte(req.Body)) } - signedHeaders, err := singer.Sign(req, seeker, "s3", "ap-northeast-1", time.Now()) + _, err := signer.Sign(req, seeker, "s3", endpoints.ApNortheast1RegionID, t) if err != nil { - fmt.Printf("error: %s\n", err) + return er } - fmt.Printf("Signed Header: %s\n", signedHeaders) + return nil +} + +func streamToByte(stream io.Reader) []byte { + buf := new(bytes.Buffer) + buf.ReadFrom(stream) + return buf.Bytes() } func (s3 *S3) resource(path string, values url.Values) string { @@ -181,10 +144,12 @@ func (s3 *S3) Put(r io.Reader, size int64, path string, md5sum []byte, contentTy req.Header.Set("Content-Type", contentType) req.Header.Set("Content-Length", fmt.Sprintf("%d", size)) - req.Header.Set("Host", req.URL.Host) req.ContentLength = size - s3.signRequest(req) + er = s3.signRequest(req) + if er != nil { + return er + } resp, er := http.DefaultClient.Do(req) if er != nil { @@ -215,7 +180,10 @@ func (s3 *S3) Get(path string) (io.ReadCloser, http.Header, error) { return nil, http.Header{}, er } - s3.signRequest(req) + er = s3.signRequest(req) + if er != nil { + return nil, http.Header{}, er + } resp, er := http.DefaultClient.Do(req) if er != nil { @@ -245,7 +213,10 @@ func (s3 *S3) Head(path string) (http.Header, error) { return http.Header{}, er } - s3.signRequest(req) + er = s3.signRequest(req) + if er != nil { + return http.Header{}, er + } resp, er := http.DefaultClient.Do(req) if er != nil { @@ -317,9 +288,10 @@ func (s3 *S3) StartMultipart(path string) (*S3Multipart, error) { return nil, er } - req.Header.Set("Host", req.URL.Host) - - s3.signRequest(req) + er = s3.signRequest(req) + if er != nil { + return nil, er + } resp, er := http.DefaultClient.Do(req) if er != nil {