From 6097da89694493508c7dba2e6ab5a59c5ab66fa9 Mon Sep 17 00:00:00 2001 From: jojoliang Date: Mon, 12 Apr 2021 21:20:30 +0800 Subject: [PATCH 01/11] prepare crypto --- ci.go | 7 ++++-- cos.go | 20 +++++++++++++++++ helper.go | 69 +++++++++++++++++++++++++++++++++++++++++++++++++++++++--- object.go | 42 ++++++++++++++++++++++++++++++----- object_part.go | 7 ++++-- 5 files changed, 132 insertions(+), 13 deletions(-) diff --git a/ci.go b/ci.go index 3958b0a..854f323 100644 --- a/ci.go +++ b/ci.go @@ -271,10 +271,13 @@ func (s *CIService) Put(ctx context.Context, name string, r io.Reader, uopt *Obj if err := CheckReaderLen(r); err != nil { return nil, nil, err } - opt := cloneObjectPutOptions(uopt) + opt := CloneObjectPutOptions(uopt) totalBytes, err := GetReaderLen(r) if err != nil && opt != nil && opt.Listener != nil { - return nil, nil, err + if opt.ContentLength == 0 { + return nil, nil, err + } + totalBytes = opt.ContentLength } if err == nil { // 与 go http 保持一致, 非bytes.Buffer/bytes.Reader/strings.Reader由用户指定ContentLength, 或使用 Chunk 上传 diff --git a/cos.go b/cos.go index f6b660c..9f71628 100644 --- a/cos.go +++ b/cos.go @@ -133,6 +133,26 @@ func NewClient(uri *BaseURL, httpClient *http.Client) *Client { return c } +type Credential struct { + SecretID string + SecretKey string + SessionToken string +} + +func (c *Client) GetCredential() *Credential { + auth, ok := c.client.Transport.(*AuthorizationTransport) + if !ok { + return nil + } + auth.rwLocker.Lock() + defer auth.rwLocker.Unlock() + return &Credential{ + SecretID: auth.SecretID, + SecretKey: auth.SecretKey, + SessionToken: auth.SessionToken, + } +} + func (c *Client) newRequest(ctx context.Context, baseURL *url.URL, uri, method string, body interface{}, optQuery interface{}, optHeader interface{}) (req *http.Request, err error) { uri, err = addURLOptions(uri, optQuery) if err != nil { diff --git a/helper.go b/helper.go index 6affc2a..34d642f 100644 --- a/helper.go +++ b/helper.go @@ -12,6 +12,7 @@ import ( "net/http" "net/url" "os" + "strconv" "strings" ) @@ -214,7 +215,7 @@ func CopyOptionsToMulti(opt *ObjectCopyOptions) *InitiateMultipartUploadOptions } // 浅拷贝ObjectPutOptions -func cloneObjectPutOptions(opt *ObjectPutOptions) *ObjectPutOptions { +func CloneObjectPutOptions(opt *ObjectPutOptions) *ObjectPutOptions { res := &ObjectPutOptions{ &ACLHeaderOptions{}, &ObjectPutHeaderOptions{}, @@ -230,8 +231,24 @@ func cloneObjectPutOptions(opt *ObjectPutOptions) *ObjectPutOptions { return res } +func CloneInitiateMultipartUploadOptions(opt *InitiateMultipartUploadOptions) *InitiateMultipartUploadOptions { + res := &InitiateMultipartUploadOptions{ + &ACLHeaderOptions{}, + &ObjectPutHeaderOptions{}, + } + if opt != nil { + if opt.ACLHeaderOptions != nil { + *res.ACLHeaderOptions = *opt.ACLHeaderOptions + } + if opt.ObjectPutHeaderOptions != nil { + *res.ObjectPutHeaderOptions = *opt.ObjectPutHeaderOptions + } + } + return res +} + // 浅拷贝ObjectUploadPartOptions -func cloneObjectUploadPartOptions(opt *ObjectUploadPartOptions) *ObjectUploadPartOptions { +func CloneObjectUploadPartOptions(opt *ObjectUploadPartOptions) *ObjectUploadPartOptions { var res ObjectUploadPartOptions if opt != nil { res = *opt @@ -239,6 +256,14 @@ func cloneObjectUploadPartOptions(opt *ObjectUploadPartOptions) *ObjectUploadPar return &res } +func CloneObjectGetOptions(opt *ObjectGetOptions) *ObjectGetOptions { + var res ObjectGetOptions + if opt != nil { + res = *opt + } + return &res +} + type RangeOptions struct { HasStart bool HasEnd bool @@ -259,7 +284,45 @@ func FormatRangeOptions(opt *RangeOptions) string { if opt.HasEnd { return fmt.Sprintf("bytes=-%v", opt.End) } - return "bytes=-" + return "" +} + +func GetRangeOptions(opt *ObjectGetOptions) (*RangeOptions, error) { + if opt == nil || opt.Range == "" { + return nil, nil + } + // bytes=M-N + slices := strings.Split(opt.Range, "=") + if len(slices) != 2 || slices[0] != "bytes" { + return nil, fmt.Errorf("Invalid Parameter Range: %v", opt.Range) + } + // byte=M-N, X-Y + fSlice := strings.Split(slices[1], ",") + rstr := fSlice[0] + + var err error + var ropt RangeOptions + sted := strings.Split(rstr, "-") + if len(sted) != 2 { + return nil, fmt.Errorf("Invalid Parameter Range: %v", opt.Range) + } + // M + if len(sted[0]) > 0 { + ropt.Start, err = strconv.ParseInt(sted[0], 10, 64) + if err != nil { + return nil, fmt.Errorf("Invalid Parameter Range: %v,err: %v", opt.Range, err) + } + ropt.HasStart = true + } + // N + if len(sted[1]) > 0 { + ropt.End, err = strconv.ParseInt(sted[1], 10, 64) + if err != nil || ropt.End == 0 { + return nil, fmt.Errorf("Invalid Parameter Range: %v,err: %v", opt.Range, err) + } + ropt.HasEnd = true + } + return &ropt, nil } var deliverHeader = map[string]bool{} diff --git a/object.go b/object.go index 1e844a3..078b92d 100644 --- a/object.go +++ b/object.go @@ -200,10 +200,13 @@ func (s *ObjectService) Put(ctx context.Context, name string, r io.Reader, uopt if err := CheckReaderLen(r); err != nil { return nil, err } - opt := cloneObjectPutOptions(uopt) + opt := CloneObjectPutOptions(uopt) totalBytes, err := GetReaderLen(r) if err != nil && opt != nil && opt.Listener != nil { - return nil, err + if opt.ContentLength == 0 { + return nil, err + } + totalBytes = opt.ContentLength } if err == nil { // 与 go http 保持一致, 非bytes.Buffer/bytes.Reader/strings.Reader由用户指定ContentLength, 或使用 Chunk 上传 @@ -630,6 +633,35 @@ func (lc *LimitedReadCloser) Close() error { return nil } +type DiscardReadCloser struct { + RC io.ReadCloser + Discard int +} + +func (drc *DiscardReadCloser) Read(data []byte) (int, error) { + n, err := drc.RC.Read(data) + if drc.Discard == 0 || n <= 0 { + return n, err + } + + if n <= drc.Discard { + drc.Discard -= n + return 0, err + } + + realLen := n - drc.Discard + copy(data[0:realLen], data[drc.Discard:n]) + drc.Discard = 0 + return realLen, err +} + +func (drc *DiscardReadCloser) Close() error { + if rc, ok := drc.RC.(io.ReadCloser); ok { + return rc.Close() + } + return nil +} + func worker(s *ObjectService, jobs <-chan *Jobs, results chan<- *Results) { for j := range jobs { j.Opt.ContentLength = j.Chunk.Size @@ -736,7 +768,6 @@ func SplitFileIntoChunks(filePath string, partSize int64) (int64, []Chunk, int, } var partNum int64 if partSize > 0 { - partSize = partSize * 1024 * 1024 partNum = stat.Size() / partSize if partNum >= 10000 { return 0, nil, 0, errors.New("Too many parts, out of 10000") @@ -855,7 +886,7 @@ func (s *ObjectService) Upload(ctx context.Context, name string, filepath string } var localcrc uint64 // 1.Get the file chunk - totalBytes, chunks, partNum, err := SplitFileIntoChunks(filepath, opt.PartSize) + totalBytes, chunks, partNum, err := SplitFileIntoChunks(filepath, opt.PartSize*1024*1024) if err != nil { return nil, nil, err } @@ -1035,7 +1066,6 @@ func (s *ObjectService) Upload(ctx context.Context, name string, filepath string func SplitSizeIntoChunks(totalBytes int64, partSize int64) ([]Chunk, int, error) { var partNum int64 if partSize > 0 { - partSize = partSize * 1024 * 1024 partNum = totalBytes / partSize if partNum >= 10000 { return nil, 0, errors.New("Too manry parts, out of 10000") @@ -1130,7 +1160,7 @@ func (s *ObjectService) Download(ctx context.Context, name string, filepath stri } // 切分 - chunks, partNum, err := SplitSizeIntoChunks(totalBytes, opt.PartSize) + chunks, partNum, err := SplitSizeIntoChunks(totalBytes, opt.PartSize*1024*1024) if err != nil { return resp, err } diff --git a/object_part.go b/object_part.go index 63e66b9..7ed0152 100644 --- a/object_part.go +++ b/object_part.go @@ -77,10 +77,13 @@ func (s *ObjectService) UploadPart(ctx context.Context, name, uploadID string, p return nil, err } // opt 不为 nil - opt := cloneObjectUploadPartOptions(uopt) + opt := CloneObjectUploadPartOptions(uopt) totalBytes, err := GetReaderLen(r) if err != nil && opt.Listener != nil { - return nil, err + if opt.ContentLength == 0 { + return nil, err + } + totalBytes = opt.ContentLength } // 分块上传不支持 Chunk 上传 if err == nil { From 7134982d66a52575ec5133d7a01d718175baf719 Mon Sep 17 00:00:00 2001 From: jojoliang Date: Mon, 12 Apr 2021 21:25:21 +0800 Subject: [PATCH 02/11] add ces-kms --- crypto/aes_ctr.go | 67 +++++++ crypto/aes_ctr_cipher.go | 138 ++++++++++++++ crypto/aes_ctr_cipher_test.go | 107 +++++++++++ crypto/cipher.go | 69 +++++++ crypto/crypto_object.go | 228 ++++++++++++++++++++++ crypto/crypto_object_part.go | 76 ++++++++ crypto/crypto_object_part_test.go | 387 ++++++++++++++++++++++++++++++++++++++ crypto/crypto_object_test.go | 385 +++++++++++++++++++++++++++++++++++++ crypto/crypto_type.go | 141 ++++++++++++++ crypto/master_kms_cipher.go | 87 +++++++++ crypto/master_kms_cipher_test.go | 91 +++++++++ example/crypto/crypto_sample.go | 305 ++++++++++++++++++++++++++++++ 12 files changed, 2081 insertions(+) create mode 100644 crypto/aes_ctr.go create mode 100644 crypto/aes_ctr_cipher.go create mode 100644 crypto/aes_ctr_cipher_test.go create mode 100644 crypto/cipher.go create mode 100644 crypto/crypto_object.go create mode 100644 crypto/crypto_object_part.go create mode 100644 crypto/crypto_object_part_test.go create mode 100644 crypto/crypto_object_test.go create mode 100644 crypto/crypto_type.go create mode 100644 crypto/master_kms_cipher.go create mode 100644 crypto/master_kms_cipher_test.go create mode 100644 example/crypto/crypto_sample.go diff --git a/crypto/aes_ctr.go b/crypto/aes_ctr.go new file mode 100644 index 0000000..1057c2c --- /dev/null +++ b/crypto/aes_ctr.go @@ -0,0 +1,67 @@ +package coscrypto + +import ( + "crypto/aes" + "crypto/cipher" + "io" +) + +type aesCtr struct { + encrypter cipher.Stream + decrypter cipher.Stream +} + +func newAesCtr(cd CipherData) (Cipher, error) { + block, err := aes.NewCipher(cd.Key) + if err != nil { + return nil, err + } + encrypter := cipher.NewCTR(block, cd.IV) + decrypter := cipher.NewCTR(block, cd.IV) + return &aesCtr{encrypter, decrypter}, nil +} + +func (c *aesCtr) Encrypt(src io.Reader) io.Reader { + reader := &ctrEncryptReader{ + encrypter: c.encrypter, + src: src, + } + return reader +} + +type ctrEncryptReader struct { + encrypter cipher.Stream + src io.Reader +} + +func (reader *ctrEncryptReader) Read(data []byte) (int, error) { + plainText := make([]byte, len(data), len(data)) + n, err := reader.src.Read(plainText) + if n > 0 { + plainText = plainText[0:n] + reader.encrypter.XORKeyStream(data, plainText) + } + return n, err +} + +func (c *aesCtr) Decrypt(src io.Reader) io.Reader { + return &ctrDecryptReader{ + decrypter: c.decrypter, + src: src, + } +} + +type ctrDecryptReader struct { + decrypter cipher.Stream + src io.Reader +} + +func (reader *ctrDecryptReader) Read(data []byte) (int, error) { + cryptoText := make([]byte, len(data), len(data)) + n, err := reader.src.Read(cryptoText) + if n > 0 { + cryptoText = cryptoText[0:n] + reader.decrypter.XORKeyStream(data, cryptoText) + } + return n, err +} diff --git a/crypto/aes_ctr_cipher.go b/crypto/aes_ctr_cipher.go new file mode 100644 index 0000000..c140cd7 --- /dev/null +++ b/crypto/aes_ctr_cipher.go @@ -0,0 +1,138 @@ +package coscrypto + +import ( + "io" +) + +const ( + aesKeySize = 32 + ivSize = 16 +) + +type aesCtrCipherBuilder struct { + MasterCipher MasterCipher +} + +type aesCtrCipher struct { + CipherData CipherData + Cipher Cipher +} + +func CreateAesCtrBuilder(cipher MasterCipher) ContentCipherBuilder { + return aesCtrCipherBuilder{MasterCipher: cipher} +} + +func (builder aesCtrCipherBuilder) createCipherData() (CipherData, error) { + var cd CipherData + var err error + err = cd.RandomKeyIv(aesKeySize, ivSize) + if err != nil { + return cd, err + } + + cd.WrapAlgorithm = builder.MasterCipher.GetWrapAlgorithm() + cd.CEKAlgorithm = AesCtrAlgorithm + cd.MatDesc = builder.MasterCipher.GetMatDesc() + + // EncryptedKey + cd.EncryptedKey, err = builder.MasterCipher.Encrypt(cd.Key) + if err != nil { + return cd, err + } + + // EncryptedIV + cd.EncryptedIV, err = builder.MasterCipher.Encrypt(cd.IV) + if err != nil { + return cd, err + } + + return cd, nil +} + +func (builder aesCtrCipherBuilder) contentCipherCD(cd CipherData) (ContentCipher, error) { + cipher, err := newAesCtr(cd) + if err != nil { + return nil, err + } + + return &aesCtrCipher{ + CipherData: cd, + Cipher: cipher, + }, nil +} + +func (builder aesCtrCipherBuilder) ContentCipher() (ContentCipher, error) { + cd, err := builder.createCipherData() + if err != nil { + return nil, err + } + return builder.contentCipherCD(cd) +} + +func (builder aesCtrCipherBuilder) ContentCipherEnv(envelope Envelope) (ContentCipher, error) { + var cd CipherData + cd.EncryptedKey = make([]byte, len(envelope.CipherKey)) + copy(cd.EncryptedKey, []byte(envelope.CipherKey)) + + plainKey, err := builder.MasterCipher.Decrypt([]byte(envelope.CipherKey)) + if err != nil { + return nil, err + } + cd.Key = make([]byte, len(plainKey)) + copy(cd.Key, plainKey) + + cd.EncryptedIV = make([]byte, len(envelope.IV)) + copy(cd.EncryptedIV, []byte(envelope.IV)) + + plainIV, err := builder.MasterCipher.Decrypt([]byte(envelope.IV)) + if err != nil { + return nil, err + } + + cd.IV = make([]byte, len(plainIV)) + copy(cd.IV, plainIV) + + cd.MatDesc = envelope.MatDesc + cd.WrapAlgorithm = envelope.WrapAlg + cd.CEKAlgorithm = envelope.CEKAlg + + return builder.contentCipherCD(cd) +} + +func (builder aesCtrCipherBuilder) GetMatDesc() string { + return builder.MasterCipher.GetMatDesc() +} + +func (cc *aesCtrCipher) EncryptContent(src io.Reader) (io.ReadCloser, error) { + reader := cc.Cipher.Encrypt(src) + return &CryptoEncrypter{Body: src, Encrypter: reader}, nil +} + +func (cc *aesCtrCipher) DecryptContent(src io.Reader) (io.ReadCloser, error) { + reader := cc.Cipher.Decrypt(src) + return &CryptoDecrypter{Body: src, Decrypter: reader}, nil +} + +func (cc *aesCtrCipher) GetCipherData() *CipherData { + return &(cc.CipherData) +} + +func (cc *aesCtrCipher) GetEncryptedLen(plainTextLen int64) int64 { + return plainTextLen +} + +func (cc *aesCtrCipher) GetAlignLen() int { + return len(cc.CipherData.IV) +} + +func (cc *aesCtrCipher) Clone(cd CipherData) (ContentCipher, error) { + cipher, err := newAesCtr(cd) + if err != nil { + return nil, err + } + + return &aesCtrCipher{ + CipherData: cd, + Cipher: cipher, + }, nil +} diff --git a/crypto/aes_ctr_cipher_test.go b/crypto/aes_ctr_cipher_test.go new file mode 100644 index 0000000..f8ebd07 --- /dev/null +++ b/crypto/aes_ctr_cipher_test.go @@ -0,0 +1,107 @@ +package coscrypto_test + +import ( + "bytes" + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "github.com/stretchr/testify/assert" + "github.com/tencentyun/cos-go-sdk-v5/crypto" + "io/ioutil" + math_rand "math/rand" +) + +type EmptyMasterCipher struct{} + +func (mc EmptyMasterCipher) Encrypt(b []byte) ([]byte, error) { + return b, nil +} +func (mc EmptyMasterCipher) Decrypt(b []byte) ([]byte, error) { + return b, nil +} +func (mc EmptyMasterCipher) GetWrapAlgorithm() string { + return "Test/EmptyWrapAlgo" +} +func (mc EmptyMasterCipher) GetMatDesc() string { + return "Empty Desc" +} + +func (s *CosTestSuite) TestCryptoObjectService_EncryptAndDecrypt() { + var masterCipher EmptyMasterCipher + builder := coscrypto.CreateAesCtrBuilder(masterCipher) + + contentCipher, err := builder.ContentCipher() + assert.Nil(s.T(), err, "CryptoObject.CreateAesCtrBuilder Failed") + + dataSize := math_rand.Int63n(1024 * 1024 * 32) + originData := make([]byte, dataSize) + rand.Read(originData) + // 加密 + r1 := bytes.NewReader(originData) + reader1, err := contentCipher.EncryptContent(r1) + assert.Nil(s.T(), err, "CryptoObject.contentCipher.Encrypt Failed") + encryptedData, err := ioutil.ReadAll(reader1) + assert.Nil(s.T(), err, "CryptoObject.Read Failed") + + // 解密 + r2 := bytes.NewReader(encryptedData) + reader2, err := contentCipher.DecryptContent(r2) + decryptedData, err := ioutil.ReadAll(reader2) + assert.Nil(s.T(), err, "CryptoObject.Read Failed") + assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData") +} + +func (s *CosTestSuite) TestCryptoObjectService_Encrypt() { + var masterCipher EmptyMasterCipher + builder := coscrypto.CreateAesCtrBuilder(masterCipher) + + contentCipher, err := builder.ContentCipher() + assert.Nil(s.T(), err, "CryptoObject.CreateAesCtrBuilder Failed") + + dataSize := math_rand.Int63n(1024 * 1024 * 32) + originData := make([]byte, dataSize) + rand.Read(originData) + + // 加密 + r := bytes.NewReader(originData) + reader, err := contentCipher.EncryptContent(r) + assert.Nil(s.T(), err, "CryptoObject.contentCipher.Encrypt Failed") + encryptedData, err := ioutil.ReadAll(reader) + assert.Nil(s.T(), err, "CryptoObject.Read Failed") + + // 直接解密 + cd := contentCipher.GetCipherData() + block, err := aes.NewCipher(cd.Key) + assert.Nil(s.T(), err, "CryptoObject.NewCipher Failed") + decrypter := cipher.NewCTR(block, cd.IV) + decryptedData := make([]byte, len(originData)) + decrypter.XORKeyStream(decryptedData, encryptedData) + assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData") +} + +func (s *CosTestSuite) TestCryptoObjectService_Decrypt() { + var masterCipher EmptyMasterCipher + builder := coscrypto.CreateAesCtrBuilder(masterCipher) + + contentCipher, err := builder.ContentCipher() + assert.Nil(s.T(), err, "CryptoObject.CreateAesCtrBuilder Failed") + dataSize := math_rand.Int63n(1024 * 1024 * 32) + originData := make([]byte, dataSize) + rand.Read(originData) + + // 直接加密 + cd := contentCipher.GetCipherData() + block, err := aes.NewCipher(cd.Key) + assert.Nil(s.T(), err, "CryptoObject.NewCipher Failed") + encrypter := cipher.NewCTR(block, cd.IV) + encryptedData := make([]byte, len(originData)) + encrypter.XORKeyStream(encryptedData, originData) + + // 解密 + r := bytes.NewReader(encryptedData) + reader, err := contentCipher.DecryptContent(r) + assert.Nil(s.T(), err, "CryptoObject.contentCipher.Encrypt Failed") + decryptedData, err := ioutil.ReadAll(reader) + assert.Nil(s.T(), err, "CryptoObject.Read Failed") + assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData") +} diff --git a/crypto/cipher.go b/crypto/cipher.go new file mode 100644 index 0000000..445f3e1 --- /dev/null +++ b/crypto/cipher.go @@ -0,0 +1,69 @@ +package coscrypto + +import ( + "io" +) + +// Cipher is interface for encryption or decryption of an object +type Cipher interface { + Encrypter + Decrypter +} + +// Encrypter is interface with only encrypt method +type Encrypter interface { + Encrypt(io.Reader) io.Reader +} + +// Decrypter is interface with only decrypt method +type Decrypter interface { + Decrypt(io.Reader) io.Reader +} + +// CryptoEncrypter provides close method for Encrypter +type CryptoEncrypter struct { + Body io.Reader + Encrypter io.Reader + isClosed bool +} + +// Close lets the CryptoEncrypter satisfy io.ReadCloser interface +func (rc *CryptoEncrypter) Close() error { + rc.isClosed = true + if closer, ok := rc.Body.(io.ReadCloser); ok { + return closer.Close() + } + return nil +} + +// Read lets the CryptoEncrypter satisfy io.ReadCloser interface +func (rc *CryptoEncrypter) Read(b []byte) (int, error) { + if rc.isClosed { + return 0, io.EOF + } + return rc.Encrypter.Read(b) +} + +// CryptoDecrypter provides close method for Decrypter +type CryptoDecrypter struct { + Body io.Reader + Decrypter io.Reader + isClosed bool +} + +// Close lets the CryptoDecrypter satisfy io.ReadCloser interface +func (rc *CryptoDecrypter) Close() error { + rc.isClosed = true + if closer, ok := rc.Body.(io.ReadCloser); ok { + return closer.Close() + } + return nil +} + +// Read lets the CryptoDecrypter satisfy io.ReadCloser interface +func (rc *CryptoDecrypter) Read(b []byte) (int, error) { + if rc.isClosed { + return 0, io.EOF + } + return rc.Decrypter.Read(b) +} diff --git a/crypto/crypto_object.go b/crypto/crypto_object.go new file mode 100644 index 0000000..903bf6a --- /dev/null +++ b/crypto/crypto_object.go @@ -0,0 +1,228 @@ +package coscrypto + +import ( + "context" + "fmt" + "github.com/tencentyun/cos-go-sdk-v5" + "io" + "net/http" + "os" + "strconv" +) + +type CryptoObjectService struct { + *cos.ObjectService + cryptoClient *CryptoClient +} + +type CryptoClient struct { + *cos.Client + Object *CryptoObjectService + ContentCipherBuilder ContentCipherBuilder + + userAgent string +} + +func NewCryptoClient(client *cos.Client, masterCipher MasterCipher) *CryptoClient { + cc := &CryptoClient{ + Client: client, + Object: &CryptoObjectService{ + client.Object, + nil, + }, + ContentCipherBuilder: CreateAesCtrBuilder(masterCipher), + } + cc.userAgent = cc.Client.UserAgent + "/" + EncryptionUaSuffix + cc.Object.cryptoClient = cc + + return cc +} + +func (s *CryptoObjectService) Put(ctx context.Context, name string, r io.Reader, opt *cos.ObjectPutOptions) (*cos.Response, error) { + cc, err := s.cryptoClient.ContentCipherBuilder.ContentCipher() + if err != nil { + return nil, err + } + reader, err := cc.EncryptContent(r) + if err != nil { + return nil, err + } + opt = cos.CloneObjectPutOptions(opt) + totalBytes, err := cos.GetReaderLen(r) + if err != nil && opt != nil && opt.Listener != nil && opt.ContentLength == 0 { + return nil, err + } + if err == nil { + if opt != nil && opt.ContentLength == 0 { + // 如果未设置Listener, 非bytes.Buffer/bytes.Reader/strings.Reader由用户指定Contength + if opt.Listener != nil || cos.IsLenReader(r) { + opt.ContentLength = totalBytes + } + } + } + if opt.XOptionHeader == nil { + opt.XOptionHeader = &http.Header{} + } + if opt.ContentMD5 != "" { + opt.XOptionHeader.Add(COSClientSideEncryptionUnencryptedContentMD5, opt.ContentMD5) + opt.ContentMD5 = "" + } + if opt.ContentLength != 0 { + opt.XOptionHeader.Add(COSClientSideEncryptionUnencryptedContentLength, strconv.FormatInt(opt.ContentLength, 10)) + opt.ContentLength = cc.GetEncryptedLen(opt.ContentLength) + } + addCryptoHeaders(opt.XOptionHeader, cc.GetCipherData()) + + return s.ObjectService.Put(ctx, name, reader, opt) +} + +func (s *CryptoObjectService) PutFromFile(ctx context.Context, name, filePath string, opt *cos.ObjectPutOptions) (resp *cos.Response, err error) { + nr := 0 + for nr < 3 { + fd, e := os.Open(filePath) + if e != nil { + err = e + return + } + resp, err = s.Put(ctx, name, fd, opt) + if err != nil { + nr++ + fd.Close() + continue + } + fd.Close() + break + } + return +} + +func (s *CryptoObjectService) Get(ctx context.Context, name string, opt *cos.ObjectGetOptions) (*cos.Response, error) { + meta, err := s.ObjectService.Head(ctx, name, nil) + if err != nil { + return meta, err + } + _isEncrypted := isEncrypted(&meta.Header) + if !_isEncrypted { + return s.ObjectService.Get(ctx, name, opt) + } + + envelope := getEnvelopeFromHeader(&meta.Header) + if !envelope.IsValid() { + return nil, fmt.Errorf("get envelope from header failed, object:%v", name) + } + encryptMatDesc := s.cryptoClient.ContentCipherBuilder.GetMatDesc() + if envelope.MatDesc != encryptMatDesc { + return nil, fmt.Errorf("provided master cipher error, want:%v, return:%v, object:%v", encryptMatDesc, envelope.MatDesc, name) + } + + cc, err := s.cryptoClient.ContentCipherBuilder.ContentCipherEnv(envelope) + if err != nil { + return nil, fmt.Errorf("get content cipher from envelope failed: %v, object:%v", err, name) + } + + optRange, err := cos.GetRangeOptions(opt) + if err != nil { + return nil, err + } + discardAlignLen := int64(0) + // Range请求 + if optRange != nil && optRange.HasStart { + // 加密block对齐 + adjustStart := adjustRangeStart(optRange.Start, int64(cc.GetAlignLen())) + discardAlignLen = optRange.Start - adjustStart + if discardAlignLen > 0 { + optRange.Start = adjustStart + opt = cos.CloneObjectGetOptions(opt) + opt.Range = cos.FormatRangeOptions(optRange) + } + + cd := cc.GetCipherData().Clone() + cd.SeekIV(uint64(adjustStart)) + cc, err = cc.Clone(cd) + if err != nil { + return nil, fmt.Errorf("ContentCipher Clone failed:%v, bject:%v", err, name) + } + } + resp, err := s.ObjectService.Get(ctx, name, opt) + if err != nil { + return resp, err + } + resp.Body, err = cc.DecryptContent(resp.Body) + if err != nil { + return resp, err + } + // 抛弃多读取的数据 + if discardAlignLen > 0 { + resp.Body = &cos.DiscardReadCloser{ + RC: resp.Body, + Discard: int(discardAlignLen), + } + } + return resp, err +} + +func (s *CryptoObjectService) GetToFile(ctx context.Context, name, localpath string, opt *cos.ObjectGetOptions) (*cos.Response, error) { + resp, err := s.Get(ctx, name, opt) + if err != nil { + return resp, err + } + defer resp.Body.Close() + + // If file exist, overwrite it + fd, err := os.OpenFile(localpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660) + if err != nil { + return resp, err + } + + _, err = io.Copy(fd, resp.Body) + fd.Close() + if err != nil { + return resp, err + } + + return resp, nil +} + +func (s *CryptoObjectService) MultiUpload(ctx context.Context, name string, filepath string, opt *cos.MultiUploadOptions) (*cos.CompleteMultipartUploadResult, *cos.Response, error) { + return s.Upload(ctx, name, filepath, opt) +} + +func (s *CryptoObjectService) Upload(ctx context.Context, name string, filepath string, opt *cos.MultiUploadOptions) (*cos.CompleteMultipartUploadResult, *cos.Response, error) { + return nil, nil, fmt.Errorf("CryptoObjectService doesn't support Upload Now") +} + +func (s *CryptoObjectService) Download(ctx context.Context, name string, filepath string, opt *cos.MultiDownloadOptions) (*cos.Response, error) { + return nil, fmt.Errorf("CryptoObjectService doesn't support Download Now") +} + +func adjustRangeStart(start int64, alignLen int64) int64 { + return (start / alignLen) * alignLen +} + +func addCryptoHeaders(header *http.Header, cd *CipherData) { + if cd.MatDesc != "" { + header.Add(COSClientSideEncryptionMatDesc, cd.MatDesc) + } + header.Add(COSClientSideEncryptionKey, string(cd.EncryptedKey)) + header.Add(COSClientSideEncryptionStart, string(cd.EncryptedIV)) + header.Add(COSClientSideEncryptionWrapAlg, cd.WrapAlgorithm) + header.Add(COSClientSideEncryptionCekAlg, cd.CEKAlgorithm) +} + +func getEnvelopeFromHeader(header *http.Header) Envelope { + var envelope Envelope + envelope.CipherKey = header.Get(COSClientSideEncryptionKey) + envelope.IV = header.Get(COSClientSideEncryptionStart) + envelope.MatDesc = header.Get(COSClientSideEncryptionMatDesc) + envelope.WrapAlg = header.Get(COSClientSideEncryptionWrapAlg) + envelope.CEKAlg = header.Get(COSClientSideEncryptionCekAlg) + return envelope +} + +func isEncrypted(header *http.Header) bool { + encryptedKey := header.Get(COSClientSideEncryptionKey) + if len(encryptedKey) > 0 { + return true + } + return false +} diff --git a/crypto/crypto_object_part.go b/crypto/crypto_object_part.go new file mode 100644 index 0000000..15561b4 --- /dev/null +++ b/crypto/crypto_object_part.go @@ -0,0 +1,76 @@ +package coscrypto + +import ( + "context" + "fmt" + "github.com/tencentyun/cos-go-sdk-v5" + "io" + "net/http" + "strconv" +) + +type CryptoContext struct { + DataSize int64 + PartSize int64 + ContentCipher ContentCipher +} + +func partSizeIsValid(partSize int64, alignLen int64) bool { + if partSize%alignLen == 0 { + return true + } + return false +} + +func (s *CryptoObjectService) InitiateMultipartUpload(ctx context.Context, name string, opt *cos.InitiateMultipartUploadOptions, cryptoCtx *CryptoContext) (*cos.InitiateMultipartUploadResult, *cos.Response, error) { + contentCipher, err := s.cryptoClient.ContentCipherBuilder.ContentCipher() + if err != nil { + return nil, nil, err + } + if !partSizeIsValid(cryptoCtx.PartSize, int64(contentCipher.GetAlignLen())) { + return nil, nil, fmt.Errorf("PartSize is invalid, it should be %v aligned", contentCipher.GetAlignLen()) + } + // 添加自定义头部 + cryptoCtx.ContentCipher = contentCipher + opt = cos.CloneInitiateMultipartUploadOptions(opt) + if opt.XOptionHeader == nil { + opt.XOptionHeader = &http.Header{} + } + if opt.ContentMD5 != "" { + opt.XOptionHeader.Add(COSClientSideEncryptionUnencryptedContentMD5, opt.ContentMD5) + opt.ContentMD5 = "" + } + opt.XOptionHeader.Add(COSClientSideEncryptionUnencryptedContentLength, strconv.FormatInt(cryptoCtx.DataSize, 10)) + addCryptoHeaders(opt.XOptionHeader, contentCipher.GetCipherData()) + + return s.ObjectService.InitiateMultipartUpload(ctx, name, opt) +} + +func (s *CryptoObjectService) UploadPart(ctx context.Context, name, uploadID string, partNumber int, r io.Reader, opt *cos.ObjectUploadPartOptions, cryptoCtx *CryptoContext) (*cos.Response, error) { + if cryptoCtx.PartSize == 0 { + return nil, fmt.Errorf("CryptoContext's PartSize is zero") + } + opt = cos.CloneObjectUploadPartOptions(opt) + if cryptoCtx.ContentCipher == nil { + return nil, fmt.Errorf("ContentCipher is nil, Please call the InitiateMultipartUpload") + } + totalBytes, err := cos.GetReaderLen(r) + if err == nil { + // 与 go http 保持一致, 非bytes.Buffer/bytes.Reader/strings.Reader需用户指定ContentLength + if opt != nil && opt.ContentLength == 0 && cos.IsLenReader(r) { + opt.ContentLength = totalBytes + } + } + cd := cryptoCtx.ContentCipher.GetCipherData().Clone() + cd.SeekIV(uint64(partNumber-1) * uint64(cryptoCtx.PartSize)) + cc, err := cryptoCtx.ContentCipher.Clone(cd) + opt.ContentLength = cc.GetEncryptedLen(opt.ContentLength) + if err != nil { + return nil, err + } + reader, err := cc.EncryptContent(r) + if err != nil { + return nil, err + } + return s.ObjectService.UploadPart(ctx, name, uploadID, partNumber, reader, opt) +} diff --git a/crypto/crypto_object_part_test.go b/crypto/crypto_object_part_test.go new file mode 100644 index 0000000..a708b2e --- /dev/null +++ b/crypto/crypto_object_part_test.go @@ -0,0 +1,387 @@ +package coscrypto_test + +import ( + "bytes" + "context" + "crypto/aes" + "crypto/cipher" + "crypto/md5" + "crypto/rand" + "encoding/base64" + "fmt" + "github.com/stretchr/testify/assert" + "github.com/tencentyun/cos-go-sdk-v5" + "github.com/tencentyun/cos-go-sdk-v5/crypto" + "io" + "io/ioutil" + math_rand "math/rand" + "net/http" + "net/url" + "os" + "sort" + "sync" + "time" +) + +func (s *CosTestSuite) TestMultiUpload_Normal() { + name := "test/ObjectPut" + time.Now().Format(time.RFC3339) + contentLength := int64(1024*1024*10 + 1) + originData := make([]byte, contentLength) + _, err := rand.Read(originData) + + cryptoCtx := coscrypto.CryptoContext{ + DataSize: contentLength, + PartSize: (contentLength / 16 / 3) * 16, + } + v, _, err := s.CClient.Object.InitiateMultipartUpload(context.Background(), name, nil, &cryptoCtx) + assert.Nil(s.T(), err, "Init Failed") + chunks, _, err := cos.SplitSizeIntoChunks(contentLength, cryptoCtx.PartSize) + assert.Nil(s.T(), err, "Split Failed") + optcom := &cos.CompleteMultipartUploadOptions{} + for _, chunk := range chunks { + opt := &cos.ObjectUploadPartOptions{ + ContentLength: chunk.Size, + } + f := bytes.NewReader(originData[chunk.OffSet : chunk.OffSet+chunk.Size]) + resp, err := s.CClient.Object.UploadPart(context.Background(), name, v.UploadID, chunk.Number, io.LimitReader(f, chunk.Size), opt, &cryptoCtx) + assert.Nil(s.T(), err, "UploadPart failed") + optcom.Parts = append(optcom.Parts, cos.Object{ + PartNumber: chunk.Number, ETag: resp.Header.Get("ETag"), + }) + } + _, _, err = s.CClient.Object.CompleteMultipartUpload(context.Background(), name, v.UploadID, optcom) + assert.Nil(s.T(), err, "Complete Failed") + + resp, err := s.CClient.Object.Get(context.Background(), name, nil) + assert.Nil(s.T(), err, "GetObject Failed") + defer resp.Body.Close() + decryptedData, _ := ioutil.ReadAll(resp.Body) + assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData") + + _, err = s.CClient.Object.Delete(context.Background(), name) + assert.Nil(s.T(), err, "DeleteObject Failed") +} + +func (s *CosTestSuite) TestMultiUpload_DecryptWithKey() { + name := "test/ObjectPut" + time.Now().Format(time.RFC3339) + contentLength := int64(1024*1024*10 + 1) + originData := make([]byte, contentLength) + _, err := rand.Read(originData) + f := bytes.NewReader(originData) + + // 分块上传 + cryptoCtx := coscrypto.CryptoContext{ + DataSize: contentLength, + PartSize: (contentLength / 16 / 3) * 16, + } + v, _, err := s.CClient.Object.InitiateMultipartUpload(context.Background(), name, nil, &cryptoCtx) + assert.Nil(s.T(), err, "Init Failed") + chunks, _, err := cos.SplitSizeIntoChunks(contentLength, cryptoCtx.PartSize) + assert.Nil(s.T(), err, "Split Failed") + optcom := &cos.CompleteMultipartUploadOptions{} + for _, chunk := range chunks { + opt := &cos.ObjectUploadPartOptions{ + ContentLength: chunk.Size, + Listener: &cos.DefaultProgressListener{}, + } + resp, err := s.CClient.Object.UploadPart(context.Background(), name, v.UploadID, chunk.Number, io.LimitReader(f, chunk.Size), opt, &cryptoCtx) + assert.Nil(s.T(), err, "UploadPart failed") + optcom.Parts = append(optcom.Parts, cos.Object{ + PartNumber: chunk.Number, ETag: resp.Header.Get("ETag"), + }) + } + _, _, err = s.CClient.Object.CompleteMultipartUpload(context.Background(), name, v.UploadID, optcom) + assert.Nil(s.T(), err, "Complete Failed") + + // 正常读取 + resp, err := s.Client.Object.Get(context.Background(), name, nil) + assert.Nil(s.T(), err, "GetObject Failed") + defer resp.Body.Close() + encryptedData, _ := ioutil.ReadAll(resp.Body) + assert.NotEqual(s.T(), bytes.Compare(encryptedData, originData), 0, "encryptedData == originData") + + // 获取解密信息 + resp, err = s.CClient.Object.Head(context.Background(), name, nil) + assert.Nil(s.T(), err, "HeadObject Failed") + cipherKey := resp.Header.Get(coscrypto.COSClientSideEncryptionKey) + cipherIV := resp.Header.Get(coscrypto.COSClientSideEncryptionStart) + key, err := s.Master.Decrypt([]byte(cipherKey)) + assert.Nil(s.T(), err, "Master Decrypt Failed") + iv, err := s.Master.Decrypt([]byte(cipherIV)) + assert.Nil(s.T(), err, "Master Decrypt Failed") + + // 手动解密 + block, err := aes.NewCipher(key) + assert.Nil(s.T(), err, "NewCipher Failed") + decrypter := cipher.NewCTR(block, iv) + decryptedData := make([]byte, len(originData)) + decrypter.XORKeyStream(decryptedData, encryptedData) + assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData") + + _, err = s.CClient.Object.Delete(context.Background(), name) + assert.Nil(s.T(), err, "DeleteObject Failed") +} + +func (s *CosTestSuite) TestMultiUpload_PutFromFile() { + name := "test/ObjectPut" + time.Now().Format(time.RFC3339) + filepath := "tmpfile" + time.Now().Format(time.RFC3339) + newfile, err := os.Create(filepath) + assert.Nil(s.T(), err, "Create File Failed") + defer os.Remove(filepath) + + contentLength := int64(1024*1024*10 + 1) + originData := make([]byte, contentLength) + _, err = rand.Read(originData) + newfile.Write(originData) + newfile.Close() + + m := md5.New() + m.Write(originData) + contentMD5 := m.Sum(nil) + cryptoCtx := coscrypto.CryptoContext{ + DataSize: contentLength, + PartSize: (contentLength / 16 / 3) * 16, + } + v, _, err := s.CClient.Object.InitiateMultipartUpload(context.Background(), name, nil, &cryptoCtx) + assert.Nil(s.T(), err, "Init Failed") + _, chunks, _, err := cos.SplitFileIntoChunks(filepath, cryptoCtx.PartSize) + assert.Nil(s.T(), err, "Split Failed") + optcom := &cos.CompleteMultipartUploadOptions{} + var wg sync.WaitGroup + var mtx sync.Mutex + for _, chunk := range chunks { + wg.Add(1) + go func(chk cos.Chunk) { + defer wg.Done() + fd, err := os.Open(filepath) + assert.Nil(s.T(), err, "Open File Failed") + opt := &cos.ObjectUploadPartOptions{ + ContentLength: chk.Size, + } + fd.Seek(chk.OffSet, os.SEEK_SET) + resp, err := s.CClient.Object.UploadPart(context.Background(), name, v.UploadID, chk.Number, io.LimitReader(fd, chk.Size), opt, &cryptoCtx) + assert.Nil(s.T(), err, "UploadPart failed") + mtx.Lock() + optcom.Parts = append(optcom.Parts, cos.Object{ + PartNumber: chk.Number, ETag: resp.Header.Get("ETag"), + }) + mtx.Unlock() + }(chunk) + } + wg.Wait() + sort.Sort(cos.ObjectList(optcom.Parts)) + _, _, err = s.CClient.Object.CompleteMultipartUpload(context.Background(), name, v.UploadID, optcom) + assert.Nil(s.T(), err, "Complete Failed") + + downfile := "downfile" + time.Now().Format(time.RFC3339) + _, err = s.CClient.Object.GetToFile(context.Background(), name, downfile, nil) + assert.Nil(s.T(), err, "GetObject Failed") + + m = md5.New() + fd, err := os.Open(downfile) + assert.Nil(s.T(), err, "Open File Failed") + defer os.Remove(downfile) + defer fd.Close() + io.Copy(m, fd) + downContentMD5 := m.Sum(nil) + assert.Equal(s.T(), bytes.Compare(contentMD5, downContentMD5), 0, "decryptData != originData") + + _, err = s.CClient.Object.Delete(context.Background(), name) + assert.Nil(s.T(), err, "DeleteObject Failed") +} + +func (s *CosTestSuite) TestMultiUpload_GetWithRange() { + name := "test/ObjectPut" + time.Now().Format(time.RFC3339) + filepath := "tmpfile" + time.Now().Format(time.RFC3339) + newfile, err := os.Create(filepath) + assert.Nil(s.T(), err, "Create File Failed") + defer os.Remove(filepath) + + contentLength := int64(1024*1024*10 + 1) + originData := make([]byte, contentLength) + _, err = rand.Read(originData) + newfile.Write(originData) + newfile.Close() + + m := md5.New() + m.Write(originData) + contentMD5 := m.Sum(nil) + cryptoCtx := coscrypto.CryptoContext{ + DataSize: contentLength, + PartSize: (contentLength / 16 / 3) * 16, + } + iniopt := &cos.InitiateMultipartUploadOptions{ + &cos.ACLHeaderOptions{ + XCosACL: "private", + }, + &cos.ObjectPutHeaderOptions{ + ContentMD5: base64.StdEncoding.EncodeToString(contentMD5), + XCosMetaXXX: &http.Header{}, + }, + } + iniopt.XCosMetaXXX.Add("x-cos-meta-isEncrypted", "true") + + v, _, err := s.CClient.Object.InitiateMultipartUpload(context.Background(), name, iniopt, &cryptoCtx) + assert.Nil(s.T(), err, "Init Failed") + _, chunks, _, err := cos.SplitFileIntoChunks(filepath, cryptoCtx.PartSize) + assert.Nil(s.T(), err, "Split Failed") + optcom := &cos.CompleteMultipartUploadOptions{} + var wg sync.WaitGroup + var mtx sync.Mutex + for _, chunk := range chunks { + wg.Add(1) + go func(chk cos.Chunk) { + defer wg.Done() + fd, err := os.Open(filepath) + assert.Nil(s.T(), err, "Open File Failed") + opt := &cos.ObjectUploadPartOptions{ + ContentLength: chk.Size, + } + fd.Seek(chk.OffSet, os.SEEK_SET) + resp, err := s.CClient.Object.UploadPart(context.Background(), name, v.UploadID, chk.Number, io.LimitReader(fd, chk.Size), opt, &cryptoCtx) + assert.Nil(s.T(), err, "UploadPart failed") + mtx.Lock() + optcom.Parts = append(optcom.Parts, cos.Object{ + PartNumber: chk.Number, ETag: resp.Header.Get("ETag"), + }) + mtx.Unlock() + }(chunk) + } + wg.Wait() + sort.Sort(cos.ObjectList(optcom.Parts)) + _, _, err = s.CClient.Object.CompleteMultipartUpload(context.Background(), name, v.UploadID, optcom) + assert.Nil(s.T(), err, "Complete Failed") + + // Range解密读取 + for i := 0; i < 10; i++ { + math_rand.Seed(time.Now().UnixNano()) + rangeStart := math_rand.Int63n(contentLength) + rangeEnd := rangeStart + math_rand.Int63n(contentLength-rangeStart) + if rangeEnd == rangeStart || rangeStart >= contentLength-1 { + continue + } + opt := &cos.ObjectGetOptions{ + Range: fmt.Sprintf("bytes=%v-%v", rangeStart, rangeEnd), + Listener: &cos.DefaultProgressListener{}, + } + resp, err := s.CClient.Object.Get(context.Background(), name, opt) + assert.Nil(s.T(), err, "GetObject Failed") + defer resp.Body.Close() + decryptedData, _ := ioutil.ReadAll(resp.Body) + assert.Equal(s.T(), bytes.Compare(originData[rangeStart:rangeEnd+1], decryptedData), 0, "decryptData != originData") + } + + opt := &cos.ObjectGetOptions{ + Listener: &cos.DefaultProgressListener{}, + } + resp, err := s.CClient.Object.Get(context.Background(), name, opt) + assert.Nil(s.T(), err, "GetObject Failed") + assert.Equal(s.T(), resp.Header.Get("x-cos-meta-isEncrypted"), "true", "meta data isn't consistent") + assert.Equal(s.T(), resp.Header.Get(coscrypto.COSClientSideEncryptionCekAlg), "AES/CTR/NoPadding", "meta data isn't consistent") + assert.Equal(s.T(), resp.Header.Get(coscrypto.COSClientSideEncryptionWrapAlg), "COS/KMS/Crypto", "meta data isn't consistent") + assert.Equal(s.T(), resp.Header.Get(coscrypto.COSClientSideEncryptionUnencryptedContentMD5), base64.StdEncoding.EncodeToString(contentMD5), "meta data isn't consistent") + defer resp.Body.Close() + decryptedData, _ := ioutil.ReadAll(resp.Body) + + assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData") + + _, err = s.CClient.Object.Delete(context.Background(), name) + assert.Nil(s.T(), err, "DeleteObject Failed") +} + +func (s *CosTestSuite) TestMultiUpload_GetWithNewClient() { + name := "test/ObjectPut" + time.Now().Format(time.RFC3339) + contentLength := int64(1024*1024*10 + 1) + originData := make([]byte, contentLength) + _, err := rand.Read(originData) + + cryptoCtx := coscrypto.CryptoContext{ + DataSize: contentLength, + PartSize: (contentLength / 16 / 3) * 16, + } + v, _, err := s.CClient.Object.InitiateMultipartUpload(context.Background(), name, nil, &cryptoCtx) + assert.Nil(s.T(), err, "Init Failed") + chunks, _, err := cos.SplitSizeIntoChunks(contentLength, cryptoCtx.PartSize) + assert.Nil(s.T(), err, "Split Failed") + optcom := &cos.CompleteMultipartUploadOptions{} + for _, chunk := range chunks { + opt := &cos.ObjectUploadPartOptions{ + ContentLength: chunk.Size, + } + f := bytes.NewReader(originData[chunk.OffSet : chunk.OffSet+chunk.Size]) + resp, err := s.CClient.Object.UploadPart(context.Background(), name, v.UploadID, chunk.Number, io.LimitReader(f, chunk.Size), opt, &cryptoCtx) + assert.Nil(s.T(), err, "UploadPart failed") + optcom.Parts = append(optcom.Parts, cos.Object{ + PartNumber: chunk.Number, ETag: resp.Header.Get("ETag"), + }) + } + _, _, err = s.CClient.Object.CompleteMultipartUpload(context.Background(), name, v.UploadID, optcom) + assert.Nil(s.T(), err, "Complete Failed") + + u, _ := url.Parse("https://" + kBucket + ".cos." + kRegion + ".myqcloud.com") + b := &cos.BaseURL{BucketURL: u} + c := cos.NewClient(b, &http.Client{ + Transport: &cos.AuthorizationTransport{ + SecretID: os.Getenv("COS_SECRETID"), + SecretKey: os.Getenv("COS_SECRETKEY"), + }, + }) + { + // 使用不同的MatDesc客户端读取, 期待错误 + material := make(map[string]string) + material["desc"] = "cos crypto suite test 2" + kmsclient, _ := coscrypto.NewKMSClient(c.GetCredential(), kRegion) + master, _ := coscrypto.CreateMasterKMS(kmsclient, os.Getenv("KMSID"), material) + client := coscrypto.NewCryptoClient(c, master) + resp, err := client.Object.Get(context.Background(), name, nil) + assert.Nil(s.T(), resp, "Get Object Failed") + assert.NotNil(s.T(), err, "Get Object Failed") + } + + { + // 使用相同的MatDesc客户端读取, 但KMSID不一样，期待正确，kms解密是不需要KMSID + material := make(map[string]string) + material["desc"] = "cos crypto suite test" + kmsclient, _ := coscrypto.NewKMSClient(c.GetCredential(), kRegion) + master, _ := coscrypto.CreateMasterKMS(kmsclient, "KMSID", material) + client := coscrypto.NewCryptoClient(c, master) + resp, err := client.Object.Get(context.Background(), name, nil) + assert.Nil(s.T(), err, "Get Object Failed") + defer resp.Body.Close() + decryptedData, _ := ioutil.ReadAll(resp.Body) + assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData") + } + + { + // 使用相同的MatDesc客户端读取, 地域不一样，期待错误 + material := make(map[string]string) + material["desc"] = "cos crypto suite test" + diffRegion := "ap-shanghai" + if diffRegion == kRegion { + diffRegion = "ap-guangzhou" + } + kmsclient, _ := coscrypto.NewKMSClient(c.GetCredential(), diffRegion) + master, _ := coscrypto.CreateMasterKMS(kmsclient, "KMSID", material) + client := coscrypto.NewCryptoClient(c, master) + resp, err := client.Object.Get(context.Background(), name, nil) + assert.Nil(s.T(), resp, "Get Object Failed") + assert.NotNil(s.T(), err, "Get Object Failed") + } + + { + // 使用相同的MatDesc和KMSID客户端读取, 期待正确 + material := make(map[string]string) + material["desc"] = "cos crypto suite test" + kmsclient, _ := coscrypto.NewKMSClient(c.GetCredential(), kRegion) + master, _ := coscrypto.CreateMasterKMS(kmsclient, os.Getenv("KMSID"), material) + client := coscrypto.NewCryptoClient(c, master) + resp, err := client.Object.Get(context.Background(), name, nil) + assert.Nil(s.T(), err, "Get Object Failed") + defer resp.Body.Close() + decryptedData, _ := ioutil.ReadAll(resp.Body) + assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData") + } + + _, err = s.CClient.Object.Delete(context.Background(), name) + assert.Nil(s.T(), err, "DeleteObject Failed") +} diff --git a/crypto/crypto_object_test.go b/crypto/crypto_object_test.go new file mode 100644 index 0000000..54bc692 --- /dev/null +++ b/crypto/crypto_object_test.go @@ -0,0 +1,385 @@ +package coscrypto_test + +import ( + "bytes" + "context" + "crypto/aes" + "crypto/cipher" + "crypto/md5" + "crypto/rand" + "encoding/base64" + "fmt" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + "github.com/tencentyun/cos-go-sdk-v5" + "github.com/tencentyun/cos-go-sdk-v5/crypto" + "io" + "io/ioutil" + math_rand "math/rand" + "net/http" + "net/url" + "os" + "testing" + "time" +) + +const ( + kAppid = 1259654469 + kBucket = "cosgosdktest-1259654469" + kRegion = "ap-guangzhou" +) + +type CosTestSuite struct { + suite.Suite + Client *cos.Client + CClient *coscrypto.CryptoClient + Master coscrypto.MasterCipher +} + +func (s *CosTestSuite) SetupSuite() { + u, _ := url.Parse("https://" + kBucket + ".cos." + kRegion + ".myqcloud.com") + b := &cos.BaseURL{BucketURL: u} + s.Client = cos.NewClient(b, &http.Client{ + Transport: &cos.AuthorizationTransport{ + SecretID: os.Getenv("COS_SECRETID"), + SecretKey: os.Getenv("COS_SECRETKEY"), + }, + }) + material := make(map[string]string) + material["desc"] = "cos crypto suite test" + kmsclient, _ := coscrypto.NewKMSClient(s.Client.GetCredential(), kRegion) + s.Master, _ = coscrypto.CreateMasterKMS(kmsclient, os.Getenv("KMSID"), material) + s.CClient = coscrypto.NewCryptoClient(s.Client, s.Master) + opt := &cos.BucketPutOptions{ + XCosACL: "public-read", + } + r, err := s.Client.Bucket.Put(context.Background(), opt) + if err != nil && r != nil && r.StatusCode == 409 { + fmt.Println("BucketAlreadyOwnedByYou") + } else if err != nil { + assert.Nil(s.T(), err, "PutBucket Failed") + } +} + +func (s *CosTestSuite) TestPutGetDeleteObject_DecryptWithKey_10MB() { + name := "test/objectPut" + time.Now().Format(time.RFC3339) + originData := make([]byte, 1024*1024*10+1) + _, err := rand.Read(originData) + f := bytes.NewReader(originData) + + // 加密存储 + _, err = s.CClient.Object.Put(context.Background(), name, f, nil) + assert.Nil(s.T(), err, "PutObject Failed") + + // 获取解密信息 + resp, err := s.CClient.Object.Head(context.Background(), name, nil) + assert.Nil(s.T(), err, "HeadObject Failed") + cipherKey := resp.Header.Get(coscrypto.COSClientSideEncryptionKey) + cipherIV := resp.Header.Get(coscrypto.COSClientSideEncryptionStart) + key, err := s.Master.Decrypt([]byte(cipherKey)) + assert.Nil(s.T(), err, "Master Decrypt Failed") + iv, err := s.Master.Decrypt([]byte(cipherIV)) + assert.Nil(s.T(), err, "Master Decrypt Failed") + + // 正常读取 + resp, err = s.Client.Object.Get(context.Background(), name, nil) + assert.Nil(s.T(), err, "GetObject Failed") + defer resp.Body.Close() + encryptedData, _ := ioutil.ReadAll(resp.Body) + assert.NotEqual(s.T(), bytes.Compare(encryptedData, originData), 0, "encryptedData == originData") + + // 手动解密 + block, err := aes.NewCipher(key) + assert.Nil(s.T(), err, "NewCipher Failed") + decrypter := cipher.NewCTR(block, iv) + decryptedData := make([]byte, len(originData)) + decrypter.XORKeyStream(decryptedData, encryptedData) + assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData") + _, err = s.CClient.Object.Delete(context.Background(), name) + assert.Nil(s.T(), err, "DeleteObject Failed") +} + +func (s *CosTestSuite) TestPutGetDeleteObject_Normal_10MB() { + name := "test/objectPut" + time.Now().Format(time.RFC3339) + originData := make([]byte, 1024*1024*10+1) + _, err := rand.Read(originData) + f := bytes.NewReader(originData) + + // 加密存储 + _, err = s.CClient.Object.Put(context.Background(), name, f, nil) + assert.Nil(s.T(), err, "PutObject Failed") + + // 解密读取 + resp, err := s.CClient.Object.Get(context.Background(), name, nil) + assert.Nil(s.T(), err, "GetObject Failed") + defer resp.Body.Close() + decryptedData, _ := ioutil.ReadAll(resp.Body) + assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData") + + _, err = s.CClient.Object.Delete(context.Background(), name) + assert.Nil(s.T(), err, "DeleteObject Failed") +} + +func (s *CosTestSuite) TestPutGetDeleteObject_ZeroFile() { + name := "test/objectPut" + time.Now().Format(time.RFC3339) + // 加密存储 + _, err := s.CClient.Object.Put(context.Background(), name, bytes.NewReader([]byte("")), nil) + assert.Nil(s.T(), err, "PutObject Failed") + + // 解密读取 + resp, err := s.CClient.Object.Get(context.Background(), name, nil) + assert.Nil(s.T(), err, "GetObject Failed") + defer resp.Body.Close() + decryptedData, _ := ioutil.ReadAll(resp.Body) + assert.Equal(s.T(), bytes.Compare([]byte(""), decryptedData), 0, "decryptData != originData") + + _, err = s.CClient.Object.Delete(context.Background(), name) + assert.Nil(s.T(), err, "DeleteObject Failed") +} + +func (s *CosTestSuite) TestPutGetDeleteObject_WithMetaData() { + name := "test/objectPut" + time.Now().Format(time.RFC3339) + originData := make([]byte, 1024*1024*10+1) + _, err := rand.Read(originData) + f := bytes.NewReader(originData) + + m := md5.New() + m.Write(originData) + contentMD5 := m.Sum(nil) + opt := &cos.ObjectPutOptions{ + &cos.ACLHeaderOptions{ + XCosACL: "private", + }, + &cos.ObjectPutHeaderOptions{ + ContentLength: 1024*1024*10 + 1, + ContentMD5: base64.StdEncoding.EncodeToString(contentMD5), + XCosMetaXXX: &http.Header{}, + }, + } + opt.XCosMetaXXX.Add("x-cos-meta-isEncrypted", "true") + // 加密存储 + _, err = s.CClient.Object.Put(context.Background(), name, f, opt) + assert.Nil(s.T(), err, "PutObject Failed") + + // 解密读取 + resp, err := s.CClient.Object.Get(context.Background(), name, nil) + assert.Nil(s.T(), err, "GetObject Failed") + defer resp.Body.Close() + decryptedData, _ := ioutil.ReadAll(resp.Body) + assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData") + assert.Equal(s.T(), resp.Header.Get("x-cos-meta-isEncrypted"), "true", "meta data isn't consistent") + assert.Equal(s.T(), resp.Header.Get(coscrypto.COSClientSideEncryptionCekAlg), "AES/CTR/NoPadding", "meta data isn't consistent") + assert.Equal(s.T(), resp.Header.Get(coscrypto.COSClientSideEncryptionWrapAlg), "COS/KMS/Crypto", "meta data isn't consistent") + assert.Equal(s.T(), resp.Header.Get(coscrypto.COSClientSideEncryptionUnencryptedContentMD5), base64.StdEncoding.EncodeToString(contentMD5), "meta data isn't consistent") + _, err = s.CClient.Object.Delete(context.Background(), name) + assert.Nil(s.T(), err, "DeleteObject Failed") +} + +func (s *CosTestSuite) TestPutGetDeleteObject_ByFile() { + name := "test/objectPut" + time.Now().Format(time.RFC3339) + filepath := "tmpfile" + time.Now().Format(time.RFC3339) + newfile, err := os.Create(filepath) + assert.Nil(s.T(), err, "Create File Failed") + defer os.Remove(filepath) + + originData := make([]byte, 1024*1024*10+1) + _, err = rand.Read(originData) + newfile.Write(originData) + newfile.Close() + + m := md5.New() + m.Write(originData) + contentMD5 := m.Sum(nil) + opt := &cos.ObjectPutOptions{ + &cos.ACLHeaderOptions{ + XCosACL: "private", + }, + &cos.ObjectPutHeaderOptions{ + ContentLength: 1024*1024*10 + 1, + ContentMD5: base64.StdEncoding.EncodeToString(contentMD5), + XCosMetaXXX: &http.Header{}, + }, + } + opt.XCosMetaXXX.Add("x-cos-meta-isEncrypted", "true") + // 加密存储 + _, err = s.CClient.Object.PutFromFile(context.Background(), name, filepath, opt) + assert.Nil(s.T(), err, "PutFromFile Failed") + + // 解密读取 + downfile := "downfile" + time.Now().Format(time.RFC3339) + resp, err := s.CClient.Object.GetToFile(context.Background(), name, downfile, nil) + assert.Nil(s.T(), err, "GetToFile Failed") + assert.Equal(s.T(), resp.Header.Get("x-cos-meta-isEncrypted"), "true", "meta data isn't consistent") + assert.Equal(s.T(), resp.Header.Get(coscrypto.COSClientSideEncryptionCekAlg), "AES/CTR/NoPadding", "meta data isn't consistent") + assert.Equal(s.T(), resp.Header.Get(coscrypto.COSClientSideEncryptionWrapAlg), "COS/KMS/Crypto", "meta data isn't consistent") + assert.Equal(s.T(), resp.Header.Get(coscrypto.COSClientSideEncryptionUnencryptedContentMD5), base64.StdEncoding.EncodeToString(contentMD5), "meta data isn't consistent") + + fd, err := os.Open(downfile) + assert.Nil(s.T(), err, "Open File Failed") + defer os.Remove(downfile) + defer fd.Close() + m = md5.New() + io.Copy(m, fd) + downContentMD5 := m.Sum(nil) + assert.Equal(s.T(), bytes.Compare(contentMD5, downContentMD5), 0, "decryptData != originData") + _, err = s.CClient.Object.Delete(context.Background(), name) + assert.Nil(s.T(), err, "DeleteObject Failed") +} + +func (s *CosTestSuite) TestPutGetDeleteObject_DecryptWithNewClient_10MB() { + name := "test/objectPut" + time.Now().Format(time.RFC3339) + originData := make([]byte, 1024*1024*10+1) + _, err := rand.Read(originData) + f := bytes.NewReader(originData) + + // 加密存储 + _, err = s.CClient.Object.Put(context.Background(), name, f, nil) + assert.Nil(s.T(), err, "PutObject Failed") + + u, _ := url.Parse("https://" + kBucket + ".cos." + kRegion + ".myqcloud.com") + b := &cos.BaseURL{BucketURL: u} + c := cos.NewClient(b, &http.Client{ + Transport: &cos.AuthorizationTransport{ + SecretID: os.Getenv("COS_SECRETID"), + SecretKey: os.Getenv("COS_SECRETKEY"), + }, + }) + { + // 使用不同的MatDesc客户端读取, 期待错误 + material := make(map[string]string) + material["desc"] = "cos crypto suite test 2" + kmsclient, _ := coscrypto.NewKMSClient(c.GetCredential(), kRegion) + master, _ := coscrypto.CreateMasterKMS(kmsclient, os.Getenv("KMSID"), material) + client := coscrypto.NewCryptoClient(c, master) + resp, err := client.Object.Get(context.Background(), name, nil) + assert.Nil(s.T(), resp, "Get Object Failed") + assert.NotNil(s.T(), err, "Get Object Failed") + } + + { + // 使用相同的MatDesc客户端读取, 但KMSID不一样，期待正确，kms解密是不需要KMSID + material := make(map[string]string) + material["desc"] = "cos crypto suite test" + kmsclient, _ := coscrypto.NewKMSClient(s.Client.GetCredential(), kRegion) + master, _ := coscrypto.CreateMasterKMS(kmsclient, "KMSID", material) + client := coscrypto.NewCryptoClient(c, master) + resp, err := client.Object.Get(context.Background(), name, nil) + assert.Nil(s.T(), err, "Get Object Failed") + defer resp.Body.Close() + decryptedData, _ := ioutil.ReadAll(resp.Body) + assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData") + } + + { + // 使用相同的MatDesc和KMSID客户端读取, 期待正确 + material := make(map[string]string) + material["desc"] = "cos crypto suite test" + kmsclient, _ := coscrypto.NewKMSClient(s.Client.GetCredential(), kRegion) + master, _ := coscrypto.CreateMasterKMS(kmsclient, os.Getenv("KMSID"), material) + client := coscrypto.NewCryptoClient(c, master) + resp, err := client.Object.Get(context.Background(), name, nil) + assert.Nil(s.T(), err, "Get Object Failed") + defer resp.Body.Close() + decryptedData, _ := ioutil.ReadAll(resp.Body) + assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData") + } + + _, err = s.CClient.Object.Delete(context.Background(), name) + assert.Nil(s.T(), err, "DeleteObject Failed") +} + +func (s *CosTestSuite) TestPutGetDeleteObject_RangeGet() { + name := "test/objectPut" + time.Now().Format(time.RFC3339) + contentLength := 1024*1024*10 + 1 + originData := make([]byte, contentLength) + _, err := rand.Read(originData) + f := bytes.NewReader(originData) + + // 加密存储 + _, err = s.CClient.Object.Put(context.Background(), name, f, nil) + assert.Nil(s.T(), err, "PutObject Failed") + + // Range解密读取 + for i := 0; i < 10; i++ { + math_rand.Seed(time.Now().UnixNano()) + rangeStart := math_rand.Intn(contentLength) + rangeEnd := rangeStart + math_rand.Intn(contentLength-rangeStart) + if rangeEnd == rangeStart || rangeStart >= contentLength-1 { + continue + } + opt := &cos.ObjectGetOptions{ + Range: fmt.Sprintf("bytes=%v-%v", rangeStart, rangeEnd), + } + resp, err := s.CClient.Object.Get(context.Background(), name, opt) + assert.Nil(s.T(), err, "GetObject Failed") + defer resp.Body.Close() + decryptedData, _ := ioutil.ReadAll(resp.Body) + assert.Equal(s.T(), bytes.Compare(originData[rangeStart:rangeEnd+1], decryptedData), 0, "decryptData != originData") + } + + // 解密读取 + resp, err := s.CClient.Object.Get(context.Background(), name, nil) + assert.Nil(s.T(), err, "GetObject Failed") + defer resp.Body.Close() + decryptedData, _ := ioutil.ReadAll(resp.Body) + assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData") + + _, err = s.CClient.Object.Delete(context.Background(), name) + assert.Nil(s.T(), err, "DeleteObject Failed") +} + +func (s *CosTestSuite) TestPutGetDeleteObject_WithListenerAndRange() { + name := "test/objectPut" + time.Now().Format(time.RFC3339) + contentLength := 1024*1024*10 + 1 + originData := make([]byte, contentLength) + _, err := rand.Read(originData) + f := bytes.NewReader(originData) + + // 加密存储 + popt := &cos.ObjectPutOptions{ + nil, + &cos.ObjectPutHeaderOptions{ + Listener: &cos.DefaultProgressListener{}, + }, + } + _, err = s.CClient.Object.Put(context.Background(), name, f, popt) + assert.Nil(s.T(), err, "PutObject Failed") + + // Range解密读取 + for i := 0; i < 10; i++ { + math_rand.Seed(time.Now().UnixNano()) + rangeStart := math_rand.Intn(contentLength) + rangeEnd := rangeStart + math_rand.Intn(contentLength-rangeStart) + if rangeEnd == rangeStart || rangeStart >= contentLength-1 { + continue + } + opt := &cos.ObjectGetOptions{ + Range: fmt.Sprintf("bytes=%v-%v", rangeStart, rangeEnd), + Listener: &cos.DefaultProgressListener{}, + } + resp, err := s.CClient.Object.Get(context.Background(), name, opt) + assert.Nil(s.T(), err, "GetObject Failed") + defer resp.Body.Close() + decryptedData, _ := ioutil.ReadAll(resp.Body) + assert.Equal(s.T(), bytes.Compare(originData[rangeStart:rangeEnd+1], decryptedData), 0, "decryptData != originData") + } + // 解密读取 + opt := &cos.ObjectGetOptions{ + Listener: &cos.DefaultProgressListener{}, + } + resp, err := s.CClient.Object.Get(context.Background(), name, opt) + assert.Nil(s.T(), err, "GetObject Failed") + defer resp.Body.Close() + decryptedData, _ := ioutil.ReadAll(resp.Body) + assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData") + + _, err = s.CClient.Object.Delete(context.Background(), name) + assert.Nil(s.T(), err, "DeleteObject Failed") +} + +func TestCosTestSuite(t *testing.T) { + suite.Run(t, new(CosTestSuite)) +} + +func (s *CosTestSuite) TearDownSuite() { +} diff --git a/crypto/crypto_type.go b/crypto/crypto_type.go new file mode 100644 index 0000000..f12557e --- /dev/null +++ b/crypto/crypto_type.go @@ -0,0 +1,141 @@ +package coscrypto + +import ( + "crypto/rand" + "encoding/binary" + "fmt" + "io" + math_rand "math/rand" + "time" +) + +const ( + COSClientSideEncryptionKey string = "x-cos-meta-client-side-encryption-key" + COSClientSideEncryptionStart = "x-cos-meta-client-side-encryption-start" + COSClientSideEncryptionCekAlg = "x-cos-meta-client-side-encryption-cek-alg" + COSClientSideEncryptionWrapAlg = "x-cos-meta-client-side-encryption-wrap-alg" + COSClientSideEncryptionMatDesc = "x-cos-meta-client-side-encryption-matdesc" + COSClientSideEncryptionUnencryptedContentLength = "x-cos-meta-client-side-encryption-unencrypted-content-length" + COSClientSideEncryptionUnencryptedContentMD5 = "x-cos-meta-client-side-encryption-unencrypted-content-md5" + COSClientSideEncryptionDataSize = "x-cos-meta-client-side-encryption-data-size" + COSClientSideEncryptionPartSize = "x-cos-meta-client-side-encryption-part-size" + COSClientUserAgent = "User-Agent" +) + +const ( + CosKmsCryptoWrap = "COS/KMS/Crypto" + AesCtrAlgorithm = "AES/CTR/NoPadding" + EncryptionUaSuffix = "COSEncryptionClient" +) + +type MasterCipher interface { + Encrypt([]byte) ([]byte, error) + Decrypt([]byte) ([]byte, error) + GetWrapAlgorithm() string + GetMatDesc() string +} + +type ContentCipherBuilder interface { + ContentCipher() (ContentCipher, error) + ContentCipherEnv(Envelope) (ContentCipher, error) + GetMatDesc() string +} + +type ContentCipher interface { + EncryptContent(io.Reader) (io.ReadCloser, error) + DecryptContent(io.Reader) (io.ReadCloser, error) + Clone(cd CipherData) (ContentCipher, error) + GetEncryptedLen(int64) int64 + GetCipherData() *CipherData + GetAlignLen() int +} + +type Envelope struct { + IV string + CipherKey string + MatDesc string + WrapAlg string + CEKAlg string + UnencryptedMD5 string + UnencryptedContentLen string +} + +func (el Envelope) IsValid() bool { + return len(el.IV) > 0 && + len(el.CipherKey) > 0 && + len(el.WrapAlg) > 0 && + len(el.CEKAlg) > 0 +} + +func (el Envelope) String() string { + return fmt.Sprintf("IV=%s&CipherKey=%s&WrapAlg=%s&CEKAlg=%s", el.IV, el.CipherKey, el.WrapAlg, el.CEKAlg) +} + +type CipherData struct { + IV []byte + Key []byte + MatDesc string + WrapAlgorithm string + CEKAlgorithm string + EncryptedIV []byte + EncryptedKey []byte +} + +func (cd *CipherData) RandomKeyIv(keyLen int, ivLen int) error { + math_rand.Seed(time.Now().UnixNano()) + + // Key + cd.Key = make([]byte, keyLen) + if _, err := io.ReadFull(rand.Reader, cd.Key); err != nil { + return err + } + + // sizeof uint64 + if ivLen < 8 { + return fmt.Errorf("ivLen:%d less than 8", ivLen) + } + + // IV: | nonce: 8 bytes | Serial number: 8 bytes | + cd.IV = make([]byte, ivLen) + if _, err := io.ReadFull(rand.Reader, cd.IV[0:ivLen-8]); err != nil { + return err + } + + // only use 4 byte,in order not to overflow when SeekIV() + randNumber := math_rand.Uint32() + cd.SetIV(uint64(randNumber)) + return nil +} + +func (cd *CipherData) SetIV(iv uint64) { + ivLen := len(cd.IV) + binary.BigEndian.PutUint64(cd.IV[ivLen-8:], iv) +} + +func (cd *CipherData) GetIV() uint64 { + ivLen := len(cd.IV) + return binary.BigEndian.Uint64(cd.IV[ivLen-8:]) +} + +func (cd *CipherData) SeekIV(startPos uint64) { + cd.SetIV(cd.GetIV() + startPos/uint64(len(cd.IV))) +} + +func (cd *CipherData) Clone() CipherData { + var cloneCd CipherData + cloneCd = *cd + + cloneCd.Key = make([]byte, len(cd.Key)) + copy(cloneCd.Key, cd.Key) + + cloneCd.IV = make([]byte, len(cd.IV)) + copy(cloneCd.IV, cd.IV) + + cloneCd.EncryptedIV = make([]byte, len(cd.EncryptedIV)) + copy(cloneCd.EncryptedIV, cd.EncryptedIV) + + cloneCd.EncryptedKey = make([]byte, len(cd.EncryptedKey)) + copy(cloneCd.EncryptedKey, cd.EncryptedKey) + + return cloneCd +} diff --git a/crypto/master_kms_cipher.go b/crypto/master_kms_cipher.go new file mode 100644 index 0000000..011458e --- /dev/null +++ b/crypto/master_kms_cipher.go @@ -0,0 +1,87 @@ +package coscrypto + +import ( + "encoding/base64" + "encoding/json" + "fmt" + + "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common" + "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/profile" + kms "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/kms/v20190118" + "github.com/tencentyun/cos-go-sdk-v5" +) + +const ( + KMSEndPoint = "kms.tencentcloudapi.com" +) + +type MasterKMSCipher struct { + Client *kms.Client + KmsId string + MatDesc string +} + +func NewKMSClient(cred *cos.Credential, region string) (*kms.Client, error) { + if cred == nil { + fmt.Errorf("credential is nil") + } + credential := common.NewTokenCredential( + cred.SecretID, + cred.SecretKey, + cred.SessionToken, + ) + cpf := profile.NewClientProfile() + cpf.HttpProfile.Endpoint = KMSEndPoint + client, err := kms.NewClient(credential, region, cpf) + return client, err +} + +func CreateMasterKMS(client *kms.Client, kmsId string, desc map[string]string) (MasterCipher, error) { + if kmsId == "" || client == nil { + return nil, fmt.Errorf("KMS ID is empty or kms client is nil") + } + var kmsCipher MasterKMSCipher + var jdesc string + if len(desc) > 0 { + bs, err := json.Marshal(desc) + if err != nil { + return nil, err + } + jdesc = string(bs) + } + kmsCipher.Client = client + kmsCipher.KmsId = kmsId + kmsCipher.MatDesc = jdesc + return &kmsCipher, nil +} + +func (kc *MasterKMSCipher) Encrypt(plaintext []byte) ([]byte, error) { + request := kms.NewEncryptRequest() + request.KeyId = common.StringPtr(kc.KmsId) + request.EncryptionContext = common.StringPtr(kc.MatDesc) + request.Plaintext = common.StringPtr(base64.StdEncoding.EncodeToString(plaintext)) + resp, err := kc.Client.Encrypt(request) + if err != nil { + return nil, err + } + return []byte(*resp.Response.CiphertextBlob), nil +} + +func (kc *MasterKMSCipher) Decrypt(ciphertext []byte) ([]byte, error) { + request := kms.NewDecryptRequest() + request.CiphertextBlob = common.StringPtr(string(ciphertext)) + request.EncryptionContext = common.StringPtr(kc.MatDesc) + resp, err := kc.Client.Decrypt(request) + if err != nil { + return nil, err + } + return base64.StdEncoding.DecodeString(*resp.Response.Plaintext) +} + +func (kc *MasterKMSCipher) GetWrapAlgorithm() string { + return CosKmsCryptoWrap +} + +func (kc *MasterKMSCipher) GetMatDesc() string { + return kc.MatDesc +} diff --git a/crypto/master_kms_cipher_test.go b/crypto/master_kms_cipher_test.go new file mode 100644 index 0000000..2fafe9a --- /dev/null +++ b/crypto/master_kms_cipher_test.go @@ -0,0 +1,91 @@ +package coscrypto_test + +import ( + "bytes" + "crypto/rand" + "encoding/base64" + "encoding/json" + "github.com/stretchr/testify/assert" + "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common" + kms "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/kms/v20190118" + "github.com/tencentyun/cos-go-sdk-v5" + "github.com/tencentyun/cos-go-sdk-v5/crypto" + "os" +) + +func (s *CosTestSuite) TestMasterKmsCipher_TestKmsClient() { + kmsclient, _ := coscrypto.NewKMSClient(&cos.Credential{ + SecretID: os.Getenv("COS_SECRETID"), + SecretKey: os.Getenv("COS_SECRETKEY"), + }, kRegion) + + originData := make([]byte, 1024) + _, err := rand.Read(originData) + + ctx := make(map[string]string) + ctx["desc"] = string(originData[:10]) + bs, _ := json.Marshal(ctx) + ctxJson := string(bs) + enReq := kms.NewEncryptRequest() + enReq.KeyId = common.StringPtr(os.Getenv("KMSID")) + enReq.EncryptionContext = common.StringPtr(ctxJson) + enReq.Plaintext = common.StringPtr(base64.StdEncoding.EncodeToString(originData)) + enResp, err := kmsclient.Encrypt(enReq) + assert.Nil(s.T(), err, "Encrypt Failed") + encryptedData := []byte(*enResp.Response.CiphertextBlob) + + deReq := kms.NewDecryptRequest() + deReq.CiphertextBlob = common.StringPtr(string(encryptedData)) + deReq.EncryptionContext = common.StringPtr(ctxJson) + deResp, err := kmsclient.Decrypt(deReq) + assert.Nil(s.T(), err, "Decrypt Failed") + decryptedData, err := base64.StdEncoding.DecodeString(*deResp.Response.Plaintext) + assert.Nil(s.T(), err, "base64 Decode Failed") + assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "originData != decryptedData") +} + +func (s *CosTestSuite) TestMasterKmsCipher_TestNormal() { + kmsclient, _ := coscrypto.NewKMSClient(&cos.Credential{ + SecretID: os.Getenv("COS_SECRETID"), + SecretKey: os.Getenv("COS_SECRETKEY"), + }, kRegion) + + desc := make(map[string]string) + desc["test"] = "TestMasterKmsCipher_TestNormal" + master, err := coscrypto.CreateMasterKMS(kmsclient, os.Getenv("KMSID"), desc) + assert.Nil(s.T(), err, "CreateMasterKMS Failed") + + originData := make([]byte, 1024) + _, err = rand.Read(originData) + + encryptedData, err := master.Encrypt(originData) + assert.Nil(s.T(), err, "Encrypt Failed") + + decryptedData, err := master.Decrypt(encryptedData) + assert.Nil(s.T(), err, "Decrypt Failed") + + assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "originData != decryptedData") +} + +func (s *CosTestSuite) TestMasterKmsCipher_TestError() { + kmsclient, _ := coscrypto.NewKMSClient(&cos.Credential{ + SecretID: os.Getenv("COS_SECRETID"), + SecretKey: os.Getenv("COS_SECRETKEY"), + }, kRegion) + + desc := make(map[string]string) + desc["test"] = "TestMasterKmsCipher_TestNormal" + master, err := coscrypto.CreateMasterKMS(kmsclient, "ErrorKMSID", desc) + assert.Nil(s.T(), err, "CreateMasterKMS Failed") + + originData := make([]byte, 1024) + _, err = rand.Read(originData) + + encryptedData, err := master.Encrypt(originData) + assert.NotNil(s.T(), err, "Encrypt Failed") + + decryptedData, err := master.Decrypt(encryptedData) + assert.NotNil(s.T(), err, "Decrypt Failed") + + assert.NotEqual(s.T(), bytes.Compare(originData, decryptedData), 0, "originData != decryptedData") +} diff --git a/example/crypto/crypto_sample.go b/example/crypto/crypto_sample.go new file mode 100644 index 0000000..54672ea --- /dev/null +++ b/example/crypto/crypto_sample.go @@ -0,0 +1,305 @@ +package main + +import ( + "bytes" + "context" + "crypto/md5" + "crypto/rand" + "fmt" + "io" + "io/ioutil" + math_rand "math/rand" + "net/http" + "net/url" + "os" + "time" + + "github.com/tencentyun/cos-go-sdk-v5" + "github.com/tencentyun/cos-go-sdk-v5/crypto" + "github.com/tencentyun/cos-go-sdk-v5/debug" +) + +func log_status(err error) { + if err == nil { + return + } + if cos.IsNotFoundError(err) { + // WARN + fmt.Println("WARN: Resource is not existed") + } else if e, ok := cos.IsCOSError(err); ok { + fmt.Printf("ERROR: Code: %v\n", e.Code) + fmt.Printf("ERROR: Message: %v\n", e.Message) + fmt.Printf("ERROR: Resource: %v\n", e.Resource) + fmt.Printf("ERROR: RequestId: %v\n", e.RequestID) + // ERROR + } else { + fmt.Printf("ERROR: %v\n", err) + // ERROR + } + os.Exit(1) +} + +func simple_put_object() { + u, _ := url.Parse("https://test-1259654469.cos.ap-guangzhou.myqcloud.com") + b := &cos.BaseURL{BucketURL: u} + c := cos.NewClient(b, &http.Client{ + Transport: &cos.AuthorizationTransport{ + SecretID: os.Getenv("COS_SECRETID"), + SecretKey: os.Getenv("COS_SECRETKEY"), + Transport: &debug.DebugRequestTransport{ + RequestHeader: true, + RequestBody: false, + ResponseHeader: true, + ResponseBody: false, + }, + }, + }) + // Case1 上传对象 + name := "test/example2" + + fmt.Println("============== simple_put_object ======================") + // 该标识信息唯一确认一个主加密密钥, 解密时，需要传入相同的标识信息 + materialDesc := make(map[string]string) + materialDesc["desc"] = "" + + // 创建KMS客户端 + kmsclient, _ := coscrypto.NewKMSClient(c.GetCredential(), "ap-guangzhou") + // 创建KMS主加密密钥，标识信息和主密钥一一对应 + kmsID := os.Getenv("KMSID") + masterCipher, _ := coscrypto.CreateMasterKMS(kmsclient, kmsID, materialDesc) + // 创建加密客户端 + client := coscrypto.NewCryptoClient(c, masterCipher) + + contentLength := 1024*1024*10 + 1 + originData := make([]byte, contentLength) + _, err := rand.Read(originData) + f := bytes.NewReader(originData) + // 加密上传 + _, err = client.Object.Put(context.Background(), name, f, nil) + log_status(err) + + math_rand.Seed(time.Now().UnixNano()) + rangeStart := math_rand.Intn(contentLength) + rangeEnd := rangeStart + math_rand.Intn(contentLength-rangeStart) + opt := &cos.ObjectGetOptions{ + Range: fmt.Sprintf("bytes=%v-%v", rangeStart, rangeEnd), + } + // 解密下载 + resp, err := client.Object.Get(context.Background(), name, opt) + log_status(err) + defer resp.Body.Close() + decryptedData, _ := ioutil.ReadAll(resp.Body) + if bytes.Compare(decryptedData, originData[rangeStart:rangeEnd+1]) != 0 { + fmt.Println("Error: encryptedData != originData") + } +} + +func simple_put_object_from_file() { + u, _ := url.Parse("https://test-1259654469.cos.ap-guangzhou.myqcloud.com") + b := &cos.BaseURL{BucketURL: u} + c := cos.NewClient(b, &http.Client{ + Transport: &cos.AuthorizationTransport{ + SecretID: os.Getenv("COS_SECRETID"), + SecretKey: os.Getenv("COS_SECRETKEY"), + Transport: &debug.DebugRequestTransport{ + RequestHeader: true, + RequestBody: false, + ResponseHeader: true, + ResponseBody: false, + }, + }, + }) + // Case1 上传对象 + name := "test/example1" + + fmt.Println("============== simple_put_object_from_file ======================") + // 该标识信息唯一确认一个主加密密钥, 解密时，需要传入相同的标识信息 + materialDesc := make(map[string]string) + materialDesc["desc"] = "" + + // 创建KMS客户端 + kmsclient, _ := coscrypto.NewKMSClient(c.GetCredential(), "ap-guangzhou") + // 创建KMS主加密密钥，标识信息和主密钥一一对应 + kmsID := os.Getenv("KMSID") + masterCipher, _ := coscrypto.CreateMasterKMS(kmsclient, kmsID, materialDesc) + // 创建加密客户端 + client := coscrypto.NewCryptoClient(c, masterCipher) + + filepath := "test" + fd, err := os.Open(filepath) + log_status(err) + defer fd.Close() + m := md5.New() + io.Copy(m, fd) + originDataMD5 := m.Sum(nil) + + // 加密上传 + _, err = client.Object.PutFromFile(context.Background(), name, filepath, nil) + log_status(err) + + // 解密下载 + _, err = client.Object.GetToFile(context.Background(), name, "./test.download", nil) + log_status(err) + + fd, err = os.Open("./test.download") + log_status(err) + defer fd.Close() + m = md5.New() + io.Copy(m, fd) + decryptedDataMD5 := m.Sum(nil) + + if bytes.Compare(decryptedDataMD5, originDataMD5) != 0 { + fmt.Println("Error: encryptedData != originData") + } +} + +func multi_put_object() { + u, _ := url.Parse("https://test-1259654469.cos.ap-guangzhou.myqcloud.com") + b := &cos.BaseURL{BucketURL: u} + c := cos.NewClient(b, &http.Client{ + Transport: &cos.AuthorizationTransport{ + SecretID: os.Getenv("COS_SECRETID"), + SecretKey: os.Getenv("COS_SECRETKEY"), + Transport: &debug.DebugRequestTransport{ + RequestHeader: true, + RequestBody: false, + ResponseHeader: true, + ResponseBody: false, + }, + }, + }) + // Case1 上传对象 + name := "test/example1" + + fmt.Println("============== multi_put_object ======================") + // 该标识信息唯一确认一个主加密密钥, 解密时，需要传入相同的标识信息 + materialDesc := make(map[string]string) + materialDesc["desc"] = "" + + // 创建KMS客户端 + kmsclient, _ := coscrypto.NewKMSClient(c.GetCredential(), "ap-guangzhou") + // 创建KMS主加密密钥，标识信息和主密钥一一对应 + kmsID := os.Getenv("KMSID") + masterCipher, _ := coscrypto.CreateMasterKMS(kmsclient, kmsID, materialDesc) + // 创建加密客户端 + client := coscrypto.NewCryptoClient(c, masterCipher) + + contentLength := int64(1024*1024*10 + 1) + originData := make([]byte, contentLength) + _, err := rand.Read(originData) + log_status(err) + + // 分块上传 + cryptoCtx := coscrypto.CryptoContext{ + DataSize: contentLength, + // 每个分块需要16字节对齐 + PartSize: (contentLength / 16 / 3) * 16, + } + v, _, err := client.Object.InitiateMultipartUpload(context.Background(), name, nil, &cryptoCtx) + log_status(err) + // 切分数据 + chunks, _, err := cos.SplitSizeIntoChunks(contentLength, cryptoCtx.PartSize) + log_status(err) + optcom := &cos.CompleteMultipartUploadOptions{} + for _, chunk := range chunks { + opt := &cos.ObjectUploadPartOptions{ + ContentLength: chunk.Size, + } + f := bytes.NewReader(originData[chunk.OffSet : chunk.OffSet+chunk.Size]) + resp, err := client.Object.UploadPart(context.Background(), name, v.UploadID, chunk.Number, f, opt, &cryptoCtx) + log_status(err) + optcom.Parts = append(optcom.Parts, cos.Object{ + PartNumber: chunk.Number, ETag: resp.Header.Get("ETag"), + }) + } + _, _, err = client.Object.CompleteMultipartUpload(context.Background(), name, v.UploadID, optcom) + log_status(err) + + resp, err := client.Object.Get(context.Background(), name, nil) + log_status(err) + defer resp.Body.Close() + decryptedData, _ := ioutil.ReadAll(resp.Body) + if bytes.Compare(decryptedData, originData) != 0 { + fmt.Println("Error: encryptedData != originData") + } +} + +func multi_put_object_from_file() { + u, _ := url.Parse("https://test-1259654469.cos.ap-guangzhou.myqcloud.com") + b := &cos.BaseURL{BucketURL: u} + c := cos.NewClient(b, &http.Client{ + Transport: &cos.AuthorizationTransport{ + SecretID: os.Getenv("COS_SECRETID"), + SecretKey: os.Getenv("COS_SECRETKEY"), + Transport: &debug.DebugRequestTransport{ + RequestHeader: true, + RequestBody: false, + ResponseHeader: true, + ResponseBody: false, + }, + }, + }) + // Case1 上传对象 + name := "test/example1" + + fmt.Println("============== multi_put_object_from_file ======================") + // 该标识信息唯一确认一个主加密密钥, 解密时，需要传入相同的标识信息 + materialDesc := make(map[string]string) + materialDesc["desc"] = "" + + // 创建KMS客户端 + kmsclient, _ := coscrypto.NewKMSClient(c.GetCredential(), "ap-guangzhou") + // 创建KMS主加密密钥，标识信息和主密钥一一对应 + kmsID := os.Getenv("KMSID") + masterCipher, _ := coscrypto.CreateMasterKMS(kmsclient, kmsID, materialDesc) + // 创建加密客户端 + client := coscrypto.NewCryptoClient(c, masterCipher) + + filepath := "test" + stat, err := os.Stat(filepath) + log_status(err) + contentLength := stat.Size() + + // 分块上传 + cryptoCtx := coscrypto.CryptoContext{ + DataSize: contentLength, + // 每个分块需要16字节对齐 + PartSize: (contentLength / 16 / 3) * 16, + } + // 切分数据 + _, chunks, _, err := cos.SplitFileIntoChunks(filepath, cryptoCtx.PartSize) + log_status(err) + + // init mulitupload + v, _, err := client.Object.InitiateMultipartUpload(context.Background(), name, nil, &cryptoCtx) + log_status(err) + + // part upload + optcom := &cos.CompleteMultipartUploadOptions{} + for _, chunk := range chunks { + fd, err := os.Open(filepath) + log_status(err) + opt := &cos.ObjectUploadPartOptions{ + ContentLength: chunk.Size, + } + fd.Seek(chunk.OffSet, os.SEEK_SET) + resp, err := client.Object.UploadPart(context.Background(), name, v.UploadID, chunk.Number, io.LimitReader(fd, chunk.Size), opt, &cryptoCtx) + log_status(err) + optcom.Parts = append(optcom.Parts, cos.Object{ + PartNumber: chunk.Number, ETag: resp.Header.Get("ETag"), + }) + } + // complete upload + _, _, err = client.Object.CompleteMultipartUpload(context.Background(), name, v.UploadID, optcom) + log_status(err) + + _, err = client.Object.GetToFile(context.Background(), name, "test.download", nil) + log_status(err) +} + +func main() { + simple_put_object() + simple_put_object_from_file() + multi_put_object() + multi_put_object_from_file() +} From 3094fa72e8d1985528cfc0c8835ac0f01e15b2c7 Mon Sep 17 00:00:00 2001 From: jojoliang Date: Mon, 12 Apr 2021 21:43:21 +0800 Subject: [PATCH 03/11] update ces-kms --- crypto/crypto_object_part.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crypto/crypto_object_part.go b/crypto/crypto_object_part.go index 15561b4..f7b1410 100644 --- a/crypto/crypto_object_part.go +++ b/crypto/crypto_object_part.go @@ -40,7 +40,10 @@ func (s *CryptoObjectService) InitiateMultipartUpload(ctx context.Context, name opt.XOptionHeader.Add(COSClientSideEncryptionUnencryptedContentMD5, opt.ContentMD5) opt.ContentMD5 = "" } - opt.XOptionHeader.Add(COSClientSideEncryptionUnencryptedContentLength, strconv.FormatInt(cryptoCtx.DataSize, 10)) + if cryptoCtx.DataSize > 0 { + opt.XOptionHeader.Add(COSClientSideEncryptionDataSize, strconv.FormatInt(cryptoCtx.DataSize, 10)) + } + opt.XOptionHeader.Add(COSClientSideEncryptionPartSize, strconv.FormatInt(cryptoCtx.PartSize, 10)) addCryptoHeaders(opt.XOptionHeader, contentCipher.GetCipherData()) return s.ObjectService.InitiateMultipartUpload(ctx, name, opt) From 6ddd32755c16dcbcf89cecb785b23d32d0199ea5 Mon Sep 17 00:00:00 2001 From: jojoliang Date: Thu, 15 Apr 2021 15:51:26 +0800 Subject: [PATCH 04/11] add Crypto UserAgent and versionid --- cos.go | 34 +++++----- crypto/crypto_object.go | 19 ++++-- crypto/crypto_object_part.go | 18 +++++ crypto/crypto_object_test.go | 155 +++++++++++++++++++++++++++++++++++++++++++ crypto/crypto_type.go | 2 +- go.mod | 1 + go.sum | 1 + helper.go | 34 +++++++++- helper_test.go | 73 ++++++++++++++++++++ object_part.go | 34 +--------- 10 files changed, 313 insertions(+), 58 deletions(-) diff --git a/cos.go b/cos.go index 9f71628..a09b62f 100644 --- a/cos.go +++ b/cos.go @@ -134,23 +134,23 @@ func NewClient(uri *BaseURL, httpClient *http.Client) *Client { } type Credential struct { - SecretID string - SecretKey string - SessionToken string + SecretID string + SecretKey string + SessionToken string } func (c *Client) GetCredential() *Credential { - auth, ok := c.client.Transport.(*AuthorizationTransport) - if !ok { - return nil - } - auth.rwLocker.Lock() - defer auth.rwLocker.Unlock() - return &Credential{ - SecretID: auth.SecretID, - SecretKey: auth.SecretKey, - SessionToken: auth.SessionToken, - } + auth, ok := c.client.Transport.(*AuthorizationTransport) + if !ok { + return nil + } + auth.rwLocker.Lock() + defer auth.rwLocker.Unlock() + return &Credential{ + SecretID: auth.SecretID, + SecretKey: auth.SecretKey, + SessionToken: auth.SessionToken, + } } func (c *Client) newRequest(ctx context.Context, baseURL *url.URL, uri, method string, body interface{}, optQuery interface{}, optHeader interface{}) (req *http.Request, err error) { @@ -195,8 +195,10 @@ func (c *Client) newRequest(ctx context.Context, baseURL *url.URL, uri, method s if contentMD5 != "" { req.Header["Content-MD5"] = []string{contentMD5} } - if c.UserAgent != "" { - req.Header.Set("User-Agent", c.UserAgent) + if v := req.Header.Get("User-Agent"); v == "" || !strings.HasPrefix(v, userAgent) { + if c.UserAgent != "" { + req.Header.Set("User-Agent", c.UserAgent) + } } if req.Header.Get("Content-Type") == "" && contentType != "" { req.Header.Set("Content-Type", contentType) diff --git a/crypto/crypto_object.go b/crypto/crypto_object.go index 903bf6a..7c8f1f7 100644 --- a/crypto/crypto_object.go +++ b/crypto/crypto_object.go @@ -71,6 +71,7 @@ func (s *CryptoObjectService) Put(ctx context.Context, name string, r io.Reader, opt.XOptionHeader.Add(COSClientSideEncryptionUnencryptedContentLength, strconv.FormatInt(opt.ContentLength, 10)) opt.ContentLength = cc.GetEncryptedLen(opt.ContentLength) } + opt.XOptionHeader.Add(UserAgent, s.cryptoClient.userAgent) addCryptoHeaders(opt.XOptionHeader, cc.GetCipherData()) return s.ObjectService.Put(ctx, name, reader, opt) @@ -96,14 +97,14 @@ func (s *CryptoObjectService) PutFromFile(ctx context.Context, name, filePath st return } -func (s *CryptoObjectService) Get(ctx context.Context, name string, opt *cos.ObjectGetOptions) (*cos.Response, error) { - meta, err := s.ObjectService.Head(ctx, name, nil) +func (s *CryptoObjectService) Get(ctx context.Context, name string, opt *cos.ObjectGetOptions, id ...string) (*cos.Response, error) { + meta, err := s.ObjectService.Head(ctx, name, nil, id...) if err != nil { return meta, err } _isEncrypted := isEncrypted(&meta.Header) if !_isEncrypted { - return s.ObjectService.Get(ctx, name, opt) + return s.ObjectService.Get(ctx, name, opt, id...) } envelope := getEnvelopeFromHeader(&meta.Header) @@ -120,6 +121,10 @@ func (s *CryptoObjectService) Get(ctx context.Context, name string, opt *cos.Obj return nil, fmt.Errorf("get content cipher from envelope failed: %v, object:%v", err, name) } + opt = cos.CloneObjectGetOptions(opt) + if opt.XOptionHeader == nil { + opt.XOptionHeader = &http.Header{} + } optRange, err := cos.GetRangeOptions(opt) if err != nil { return nil, err @@ -132,7 +137,6 @@ func (s *CryptoObjectService) Get(ctx context.Context, name string, opt *cos.Obj discardAlignLen = optRange.Start - adjustStart if discardAlignLen > 0 { optRange.Start = adjustStart - opt = cos.CloneObjectGetOptions(opt) opt.Range = cos.FormatRangeOptions(optRange) } @@ -143,7 +147,8 @@ func (s *CryptoObjectService) Get(ctx context.Context, name string, opt *cos.Obj return nil, fmt.Errorf("ContentCipher Clone failed:%v, bject:%v", err, name) } } - resp, err := s.ObjectService.Get(ctx, name, opt) + opt.XOptionHeader.Add(UserAgent, s.cryptoClient.userAgent) + resp, err := s.ObjectService.Get(ctx, name, opt, id...) if err != nil { return resp, err } @@ -161,8 +166,8 @@ func (s *CryptoObjectService) Get(ctx context.Context, name string, opt *cos.Obj return resp, err } -func (s *CryptoObjectService) GetToFile(ctx context.Context, name, localpath string, opt *cos.ObjectGetOptions) (*cos.Response, error) { - resp, err := s.Get(ctx, name, opt) +func (s *CryptoObjectService) GetToFile(ctx context.Context, name, localpath string, opt *cos.ObjectGetOptions, id ...string) (*cos.Response, error) { + resp, err := s.Get(ctx, name, opt, id...) if err != nil { return resp, err } diff --git a/crypto/crypto_object_part.go b/crypto/crypto_object_part.go index f7b1410..cc34463 100644 --- a/crypto/crypto_object_part.go +++ b/crypto/crypto_object_part.go @@ -44,6 +44,7 @@ func (s *CryptoObjectService) InitiateMultipartUpload(ctx context.Context, name opt.XOptionHeader.Add(COSClientSideEncryptionDataSize, strconv.FormatInt(cryptoCtx.DataSize, 10)) } opt.XOptionHeader.Add(COSClientSideEncryptionPartSize, strconv.FormatInt(cryptoCtx.PartSize, 10)) + opt.XOptionHeader.Add(UserAgent, s.cryptoClient.userAgent) addCryptoHeaders(opt.XOptionHeader, contentCipher.GetCipherData()) return s.ObjectService.InitiateMultipartUpload(ctx, name, opt) @@ -54,6 +55,10 @@ func (s *CryptoObjectService) UploadPart(ctx context.Context, name, uploadID str return nil, fmt.Errorf("CryptoContext's PartSize is zero") } opt = cos.CloneObjectUploadPartOptions(opt) + if opt.XOptionHeader == nil { + opt.XOptionHeader = &http.Header{} + } + opt.XOptionHeader.Add(UserAgent, s.cryptoClient.userAgent) if cryptoCtx.ContentCipher == nil { return nil, fmt.Errorf("ContentCipher is nil, Please call the InitiateMultipartUpload") } @@ -77,3 +82,16 @@ func (s *CryptoObjectService) UploadPart(ctx context.Context, name, uploadID str } return s.ObjectService.UploadPart(ctx, name, uploadID, partNumber, reader, opt) } + +func (s *CryptoObjectService) CompleteMultipartUpload(ctx context.Context, name, uploadID string, opt *cos.CompleteMultipartUploadOptions) (*cos.CompleteMultipartUploadResult, *cos.Response, error) { + opt = cos.CloneCompleteMultipartUploadOptions(opt) + if opt.XOptionHeader == nil { + opt.XOptionHeader = &http.Header{} + } + opt.XOptionHeader.Add(UserAgent, s.cryptoClient.userAgent) + return s.ObjectService.CompleteMultipartUpload(ctx, name, uploadID, opt) +} + +func (s *CryptoObjectService) CopyPart(ctx context.Context, name, uploadID string, partNumber int, sourceURL string, opt *cos.ObjectCopyPartOptions) (*cos.CopyPartResult, *cos.Response, error) { + return nil, nil, fmt.Errorf("CryptoObjectService doesn't support CopyPart") +} diff --git a/crypto/crypto_object_test.go b/crypto/crypto_object_test.go index 54bc692..4c24228 100644 --- a/crypto/crypto_object_test.go +++ b/crypto/crypto_object_test.go @@ -120,6 +120,45 @@ func (s *CosTestSuite) TestPutGetDeleteObject_Normal_10MB() { assert.Nil(s.T(), err, "DeleteObject Failed") } +func (s *CosTestSuite) TestPutGetDeleteObject_VersionID() { + name := "test/objectPut" + time.Now().Format(time.RFC3339) + originData := make([]byte, 1024*1024*10+1) + _, err := rand.Read(originData) + f := bytes.NewReader(originData) + + opt := &cos.BucketPutVersionOptions{ + Status: "Enabled", + } + _, err = s.CClient.Bucket.PutVersioning(context.Background(), opt) + assert.Nil(s.T(), err, "PutVersioning Failed") + time.Sleep(3 * time.Second) + + // 加密存储 + resp, err := s.CClient.Object.Put(context.Background(), name, f, nil) + assert.Nil(s.T(), err, "PutObject Failed") + versionId := resp.Header.Get("x-cos-version-id") + + _, err = s.CClient.Object.Delete(context.Background(), name) + assert.Nil(s.T(), err, "DeleteObject Failed") + + // 解密读取 + resp, err = s.CClient.Object.Get(context.Background(), name, nil, versionId) + assert.Nil(s.T(), err, "GetObject Failed") + defer resp.Body.Close() + decryptedData, _ := ioutil.ReadAll(resp.Body) + assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData") + + delopt := &cos.ObjectDeleteOptions{ + VersionId: versionId, + } + _, err = s.CClient.Object.Delete(context.Background(), name, delopt) + assert.Nil(s.T(), err, "DeleteObject Failed") + + opt.Status = "Suspended" + _, err = s.CClient.Bucket.PutVersioning(context.Background(), opt) + assert.Nil(s.T(), err, "PutVersioning Failed") +} + func (s *CosTestSuite) TestPutGetDeleteObject_ZeroFile() { name := "test/objectPut" + time.Now().Format(time.RFC3339) // 加密存储 @@ -377,6 +416,122 @@ func (s *CosTestSuite) TestPutGetDeleteObject_WithListenerAndRange() { assert.Nil(s.T(), err, "DeleteObject Failed") } +func (s *CosTestSuite) TestPutGetDeleteObject_Copy() { + name := "test/objectPut" + time.Now().Format(time.RFC3339) + contentLength := 1024*1024*10 + 1 + originData := make([]byte, contentLength) + _, err := rand.Read(originData) + f := bytes.NewReader(originData) + + // 加密存储 + popt := &cos.ObjectPutOptions{ + nil, + &cos.ObjectPutHeaderOptions{ + Listener: &cos.DefaultProgressListener{}, + }, + } + resp, err := s.CClient.Object.Put(context.Background(), name, f, popt) + assert.Nil(s.T(), err, "PutObject Failed") + encryptedDataCRC := resp.Header.Get("x-cos-hash-crc64ecma") + time.Sleep(3 * time.Second) + sourceURL := fmt.Sprintf("%s/%s", s.CClient.BaseURL.BucketURL.Host, name) + { + // x-cos-metadata-directive必须为Copy，否则丢失加密信息，无法解密 + dest := "test/ObjectCopy1" + time.Now().Format(time.RFC3339) + res, _, err := s.CClient.Object.Copy(context.Background(), dest, sourceURL, nil) + assert.Nil(s.T(), err, "ObjectCopy Failed") + assert.Equal(s.T(), encryptedDataCRC, res.CRC64, "CRC isn't consistent, return:%v, want:%v", res.CRC64, encryptedDataCRC) + + // Range解密读取 + for i := 0; i < 3; i++ { + math_rand.Seed(time.Now().UnixNano()) + rangeStart := math_rand.Intn(contentLength) + rangeEnd := rangeStart + math_rand.Intn(contentLength-rangeStart) + if rangeEnd == rangeStart || rangeStart >= contentLength-1 { + continue + } + opt := &cos.ObjectGetOptions{ + Range: fmt.Sprintf("bytes=%v-%v", rangeStart, rangeEnd), + Listener: &cos.DefaultProgressListener{}, + } + resp, err := s.CClient.Object.Get(context.Background(), dest, opt) + assert.Nil(s.T(), err, "GetObject Failed") + defer resp.Body.Close() + decryptedData, _ := ioutil.ReadAll(resp.Body) + assert.Equal(s.T(), bytes.Compare(originData[rangeStart:rangeEnd+1], decryptedData), 0, "decryptData != originData") + } + // 解密读取 + resp, err := s.CClient.Object.Get(context.Background(), dest, nil) + assert.Nil(s.T(), err, "GetObject Failed") + defer resp.Body.Close() + decryptedData, _ := ioutil.ReadAll(resp.Body) + assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData") + + _, err = s.CClient.Object.Delete(context.Background(), dest) + assert.Nil(s.T(), err, "DeleteObject Failed") + } + { + // x-cos-metadata-directive必须为Copy，否则丢失加密信息，无法解密 + opt := &cos.ObjectCopyOptions{ + &cos.ObjectCopyHeaderOptions{ + XCosMetadataDirective: "Replaced", + }, + nil, + } + dest := "test/ObjectCopy2" + time.Now().Format(time.RFC3339) + res, _, err := s.CClient.Object.Copy(context.Background(), dest, sourceURL, opt) + assert.Nil(s.T(), err, "ObjectCopy Failed") + assert.Equal(s.T(), encryptedDataCRC, res.CRC64, "CRC isn't consistent, return:%v, want:%v", res.CRC64, encryptedDataCRC) + + // 解密读取 + resp, err := s.CClient.Object.Get(context.Background(), dest, nil) + assert.Nil(s.T(), err, "GetObject Failed") + defer resp.Body.Close() + decryptedData, _ := ioutil.ReadAll(resp.Body) + assert.NotEqual(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData") + + _, err = s.CClient.Object.Delete(context.Background(), dest) + assert.Nil(s.T(), err, "DeleteObject Failed") + } + { + // MultiCopy若是分块拷贝，则无法拷贝元数据 + dest := "test/ObjectCopy3" + time.Now().Format(time.RFC3339) + res, _, err := s.CClient.Object.MultiCopy(context.Background(), dest, sourceURL, nil) + assert.Nil(s.T(), err, "ObjectMultiCopy Failed") + assert.Equal(s.T(), encryptedDataCRC, res.CRC64, "CRC isn't consistent, return:%v, want:%v", res.CRC64, encryptedDataCRC) + // Range解密读取 + for i := 0; i < 3; i++ { + math_rand.Seed(time.Now().UnixNano()) + rangeStart := math_rand.Intn(contentLength) + rangeEnd := rangeStart + math_rand.Intn(contentLength-rangeStart) + if rangeEnd == rangeStart || rangeStart >= contentLength-1 { + continue + } + opt := &cos.ObjectGetOptions{ + Range: fmt.Sprintf("bytes=%v-%v", rangeStart, rangeEnd), + Listener: &cos.DefaultProgressListener{}, + } + resp, err := s.CClient.Object.Get(context.Background(), dest, opt) + assert.Nil(s.T(), err, "GetObject Failed") + defer resp.Body.Close() + decryptedData, _ := ioutil.ReadAll(resp.Body) + assert.Equal(s.T(), bytes.Compare(originData[rangeStart:rangeEnd+1], decryptedData), 0, "decryptData != originData") + } + // 解密读取 + resp, err := s.CClient.Object.Get(context.Background(), dest, nil) + assert.Nil(s.T(), err, "GetObject Failed") + defer resp.Body.Close() + decryptedData, _ := ioutil.ReadAll(resp.Body) + assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData") + + _, err = s.CClient.Object.Delete(context.Background(), dest) + assert.Nil(s.T(), err, "DeleteObject Failed") + } + + _, err = s.CClient.Object.Delete(context.Background(), name) + assert.Nil(s.T(), err, "DeleteObject Failed") +} + func TestCosTestSuite(t *testing.T) { suite.Run(t, new(CosTestSuite)) } diff --git a/crypto/crypto_type.go b/crypto/crypto_type.go index f12557e..5b13a55 100644 --- a/crypto/crypto_type.go +++ b/crypto/crypto_type.go @@ -19,7 +19,7 @@ const ( COSClientSideEncryptionUnencryptedContentMD5 = "x-cos-meta-client-side-encryption-unencrypted-content-md5" COSClientSideEncryptionDataSize = "x-cos-meta-client-side-encryption-data-size" COSClientSideEncryptionPartSize = "x-cos-meta-client-side-encryption-part-size" - COSClientUserAgent = "User-Agent" + UserAgent = "User-Agent" ) const ( diff --git a/go.mod b/go.mod index c01628b..6fd793d 100644 --- a/go.mod +++ b/go.mod @@ -8,4 +8,5 @@ require ( github.com/google/uuid v1.1.1 github.com/mozillazg/go-httpheader v0.2.1 github.com/stretchr/testify v1.3.0 + github.com/tencentcloud/tencentcloud-sdk-go v3.0.233+incompatible ) diff --git a/go.sum b/go.sum index ab8a08d..87103e7 100644 --- a/go.sum +++ b/go.sum @@ -13,3 +13,4 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/tencentcloud/tencentcloud-sdk-go v3.0.233+incompatible/go.mod h1:0PfYow01SHPMhKY31xa+EFz2RStxIqj6JFAJS+IkCi4= diff --git a/helper.go b/helper.go index 34d642f..85de9cd 100644 --- a/helper.go +++ b/helper.go @@ -184,6 +184,19 @@ func CheckReaderLen(reader io.Reader) error { return errors.New("The single object size you upload can not be larger than 5GB") } +func cloneHeader(opt *http.Header) *http.Header { + if opt == nil { + return nil + } + h := make(http.Header, len(*opt)) + for k, vv := range *opt { + vv2 := make([]string, len(vv)) + copy(vv2, vv) + h[k] = vv2 + } + return &h +} + func CopyOptionsToMulti(opt *ObjectCopyOptions) *InitiateMultipartUploadOptions { if opt == nil { return nil @@ -214,7 +227,6 @@ func CopyOptionsToMulti(opt *ObjectCopyOptions) *InitiateMultipartUploadOptions return optini } -// 浅拷贝ObjectPutOptions func CloneObjectPutOptions(opt *ObjectPutOptions) *ObjectPutOptions { res := &ObjectPutOptions{ &ACLHeaderOptions{}, @@ -226,6 +238,8 @@ func CloneObjectPutOptions(opt *ObjectPutOptions) *ObjectPutOptions { } if opt.ObjectPutHeaderOptions != nil { *res.ObjectPutHeaderOptions = *opt.ObjectPutHeaderOptions + res.XCosMetaXXX = cloneHeader(opt.XCosMetaXXX) + res.XOptionHeader = cloneHeader(opt.XOptionHeader) } } return res @@ -242,16 +256,18 @@ func CloneInitiateMultipartUploadOptions(opt *InitiateMultipartUploadOptions) *I } if opt.ObjectPutHeaderOptions != nil { *res.ObjectPutHeaderOptions = *opt.ObjectPutHeaderOptions + res.XCosMetaXXX = cloneHeader(opt.XCosMetaXXX) + res.XOptionHeader = cloneHeader(opt.XOptionHeader) } } return res } -// 浅拷贝ObjectUploadPartOptions func CloneObjectUploadPartOptions(opt *ObjectUploadPartOptions) *ObjectUploadPartOptions { var res ObjectUploadPartOptions if opt != nil { res = *opt + res.XOptionHeader = cloneHeader(opt.XOptionHeader) } return &res } @@ -260,10 +276,24 @@ func CloneObjectGetOptions(opt *ObjectGetOptions) *ObjectGetOptions { var res ObjectGetOptions if opt != nil { res = *opt + res.XOptionHeader = cloneHeader(opt.XOptionHeader) } return &res } +func CloneCompleteMultipartUploadOptions(opt *CompleteMultipartUploadOptions) *CompleteMultipartUploadOptions { + var res CompleteMultipartUploadOptions + if opt != nil { + res.XMLName = opt.XMLName + if len(opt.Parts) > 0 { + res.Parts = make([]Object, len(opt.Parts)) + copy(res.Parts, opt.Parts) + } + res.XOptionHeader = cloneHeader(opt.XOptionHeader) + } + return &res +} + type RangeOptions struct { HasStart bool HasEnd bool diff --git a/helper_test.go b/helper_test.go index 75ef294..b10c723 100644 --- a/helper_test.go +++ b/helper_test.go @@ -1,7 +1,10 @@ package cos import ( + "encoding/xml" "fmt" + "net/http" + "reflect" "testing" ) @@ -22,3 +25,73 @@ func Test_calMD5Digest(t *testing.T) { t.Errorf("calMD5Digest request md5: %+v, want %+v", got, want) } } + +func Test_cloneHeader(t *testing.T) { + ori := http.Header{} + opt := &ori + opt.Add("TestHeader1", "h1") + opt.Add("TestHeader1", "h2") + res := cloneHeader(opt) + if !reflect.DeepEqual(res, opt) { + t.Errorf("cloneHeader, returned:%+v, want:%+v", res, opt) + } + if !reflect.DeepEqual(ori, *opt) { + t.Errorf("cloneHeader, returned:%+v, want:%+v", *opt, ori) + } + res.Add("cloneHeader1", "c1") + res.Add("cloneHeader2", "c2") + if v := opt.Get("cloneHeader1"); v != "" { + t.Errorf("cloneHeader, returned:%+v, want:%+v", res, opt) + } + if v := opt.Get("cloneHeader2"); v != "" { + t.Errorf("cloneHeader, returned:%+v, want:%+v", res, opt) + } + opt = &http.Header{} + res = cloneHeader(opt) + if !reflect.DeepEqual(res, opt) { + t.Errorf("cloneHeader, returned:%+v, want:%+v", res, opt) + } +} + +func Test_CloneCompleteMultipartUploadOptions(t *testing.T) { + ori := CompleteMultipartUploadOptions{ + XMLName: xml.Name{Local: "CompleteMultipartUploadResult"}, + Parts: []Object{ + { + Key: "Key1", + ETag: "Etag1", + }, + { + Key: "Key2", + ETag: "Etag2", + }, + }, + XOptionHeader: &http.Header{}, + } + ori.XOptionHeader.Add("Test", "value") + opt := &ori + res := CloneCompleteMultipartUploadOptions(opt) + if !reflect.DeepEqual(res, opt) { + t.Errorf("CloneCompleteMultipartUploadOptions, returned:%+v,want:%+v", res, opt) + } + if !reflect.DeepEqual(ori, *opt) { + t.Errorf("CloneCompleteMultipartUploadOptions, returned:%+v,want:%+v", *opt, ori) + } + res.XOptionHeader.Add("TestClone", "value") + if v := opt.XOptionHeader.Get("TestClone"); v != "" { + t.Errorf("CloneCompleteMultipartUploadOptions, returned:%+v,want:%+v", res, opt) + } + opt = &CompleteMultipartUploadOptions{} + res = CloneCompleteMultipartUploadOptions(opt) + if !reflect.DeepEqual(res, opt) { + t.Errorf("CloneCompleteMultipartUploadOptions, returned:%+v,want:%+v", res, opt) + } + res.Parts = append(res.Parts, Object{Key: "K", ETag: "T"}) + if len(opt.Parts) > 0 { + t.Errorf("CloneCompleteMultipartUploadOptions Failed") + } + if reflect.DeepEqual(res, opt) { + t.Errorf("CloneCompleteMultipartUploadOptions, returned:%+v,want:%+v", res, opt) + } + +} diff --git a/object_part.go b/object_part.go index 7ed0152..2a94675 100644 --- a/object_part.go +++ b/object_part.go @@ -402,37 +402,7 @@ func (s *ObjectService) innerHead(ctx context.Context, sourceURL string, opt *Ob return } -func SplitCopyFileIntoChunks(totalBytes int64, partSize int64) ([]Chunk, int, error) { - var partNum int64 - if partSize > 0 { - partSize = partSize * 1024 * 1024 - partNum = totalBytes / partSize - if partNum >= 10000 { - return nil, 0, errors.New("Too many parts, out of 10000") - } - } else { - partNum, partSize = DividePart(totalBytes, 128) - } - - var chunks []Chunk - var chunk = Chunk{} - for i := int64(0); i < partNum; i++ { - chunk.Number = int(i + 1) - chunk.OffSet = i * partSize - chunk.Size = partSize - chunks = append(chunks, chunk) - } - - if totalBytes%partSize > 0 { - chunk.Number = len(chunks) + 1 - chunk.OffSet = int64(len(chunks)) * partSize - chunk.Size = totalBytes % partSize - chunks = append(chunks, chunk) - partNum++ - } - return chunks, int(partNum), nil -} - +// 如果源对象大于5G，则采用分块复制的方式进行拷贝，此时源对象的元信息如果COPY func (s *ObjectService) MultiCopy(ctx context.Context, name string, sourceURL string, opt *MultiCopyOptions, id ...string) (*ObjectCopyResult, *Response, error) { resp, err := s.innerHead(ctx, sourceURL, nil, id) if err != nil { @@ -455,7 +425,7 @@ func (s *ObjectService) MultiCopy(ctx context.Context, name string, sourceURL st if opt == nil { opt = &MultiCopyOptions{} } - chunks, partNum, err := SplitCopyFileIntoChunks(totalBytes, opt.PartSize) + chunks, partNum, err := SplitSizeIntoChunks(totalBytes, opt.PartSize*1024*1024) if err != nil { return nil, nil, err } From d7608ad3bc2e46f2213e4c2faaf466cd802cf24a Mon Sep 17 00:00:00 2001 From: jojoliang Date: Wed, 28 Apr 2021 21:34:17 +0800 Subject: [PATCH 05/11] update crypto sample --- example/crypto/crypto_sample.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/crypto/crypto_sample.go b/example/crypto/crypto_sample.go index 54672ea..e115ecc 100644 --- a/example/crypto/crypto_sample.go +++ b/example/crypto/crypto_sample.go @@ -283,7 +283,7 @@ func multi_put_object_from_file() { ContentLength: chunk.Size, } fd.Seek(chunk.OffSet, os.SEEK_SET) - resp, err := client.Object.UploadPart(context.Background(), name, v.UploadID, chunk.Number, io.LimitReader(fd, chunk.Size), opt, &cryptoCtx) + resp, err := client.Object.UploadPart(context.Background(), name, v.UploadID, chunk.Number, cos.LimitReadCloser(fd, chunk.Size), opt, &cryptoCtx) log_status(err) optcom.Parts = append(optcom.Parts, cos.Object{ PartNumber: chunk.Number, ETag: resp.Header.Get("ETag"), From 06f101d9e3de4cf038c2a50fc87c4f2830e397fe Mon Sep 17 00:00:00 2001 From: jojoliang Date: Wed, 28 Apr 2021 21:37:52 +0800 Subject: [PATCH 06/11] update crypto wrap meta --- crypto/crypto_type.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/crypto_type.go b/crypto/crypto_type.go index 5b13a55..c954c30 100644 --- a/crypto/crypto_type.go +++ b/crypto/crypto_type.go @@ -23,7 +23,7 @@ const ( ) const ( - CosKmsCryptoWrap = "COS/KMS/Crypto" + CosKmsCryptoWrap = "KMS/TencentCloud" AesCtrAlgorithm = "AES/CTR/NoPadding" EncryptionUaSuffix = "COSEncryptionClient" ) From 0edf665bf13714f57397b920ac613221a9da4209 Mon Sep 17 00:00:00 2001 From: jojoliang Date: Thu, 29 Apr 2021 16:54:30 +0800 Subject: [PATCH 07/11] update crypto test --- crypto/crypto_object_part_test.go | 4 ++-- crypto/crypto_object_test.go | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/crypto/crypto_object_part_test.go b/crypto/crypto_object_part_test.go index a708b2e..75a05f5 100644 --- a/crypto/crypto_object_part_test.go +++ b/crypto/crypto_object_part_test.go @@ -277,8 +277,8 @@ func (s *CosTestSuite) TestMultiUpload_GetWithRange() { resp, err := s.CClient.Object.Get(context.Background(), name, opt) assert.Nil(s.T(), err, "GetObject Failed") assert.Equal(s.T(), resp.Header.Get("x-cos-meta-isEncrypted"), "true", "meta data isn't consistent") - assert.Equal(s.T(), resp.Header.Get(coscrypto.COSClientSideEncryptionCekAlg), "AES/CTR/NoPadding", "meta data isn't consistent") - assert.Equal(s.T(), resp.Header.Get(coscrypto.COSClientSideEncryptionWrapAlg), "COS/KMS/Crypto", "meta data isn't consistent") + assert.Equal(s.T(), resp.Header.Get(coscrypto.COSClientSideEncryptionCekAlg), coscrypto.AesCtrAlgorithm, "meta data isn't consistent") + assert.Equal(s.T(), resp.Header.Get(coscrypto.COSClientSideEncryptionWrapAlg), coscrypto.CosKmsCryptoWrap, "meta data isn't consistent") assert.Equal(s.T(), resp.Header.Get(coscrypto.COSClientSideEncryptionUnencryptedContentMD5), base64.StdEncoding.EncodeToString(contentMD5), "meta data isn't consistent") defer resp.Body.Close() decryptedData, _ := ioutil.ReadAll(resp.Body) diff --git a/crypto/crypto_object_test.go b/crypto/crypto_object_test.go index 4c24228..a662c71 100644 --- a/crypto/crypto_object_test.go +++ b/crypto/crypto_object_test.go @@ -207,8 +207,8 @@ func (s *CosTestSuite) TestPutGetDeleteObject_WithMetaData() { decryptedData, _ := ioutil.ReadAll(resp.Body) assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData") assert.Equal(s.T(), resp.Header.Get("x-cos-meta-isEncrypted"), "true", "meta data isn't consistent") - assert.Equal(s.T(), resp.Header.Get(coscrypto.COSClientSideEncryptionCekAlg), "AES/CTR/NoPadding", "meta data isn't consistent") - assert.Equal(s.T(), resp.Header.Get(coscrypto.COSClientSideEncryptionWrapAlg), "COS/KMS/Crypto", "meta data isn't consistent") + assert.Equal(s.T(), resp.Header.Get(coscrypto.COSClientSideEncryptionCekAlg), coscrypto.AesCtrAlgorithm, "meta data isn't consistent") + assert.Equal(s.T(), resp.Header.Get(coscrypto.COSClientSideEncryptionWrapAlg), coscrypto.CosKmsCryptoWrap, "meta data isn't consistent") assert.Equal(s.T(), resp.Header.Get(coscrypto.COSClientSideEncryptionUnencryptedContentMD5), base64.StdEncoding.EncodeToString(contentMD5), "meta data isn't consistent") _, err = s.CClient.Object.Delete(context.Background(), name) assert.Nil(s.T(), err, "DeleteObject Failed") @@ -249,8 +249,8 @@ func (s *CosTestSuite) TestPutGetDeleteObject_ByFile() { resp, err := s.CClient.Object.GetToFile(context.Background(), name, downfile, nil) assert.Nil(s.T(), err, "GetToFile Failed") assert.Equal(s.T(), resp.Header.Get("x-cos-meta-isEncrypted"), "true", "meta data isn't consistent") - assert.Equal(s.T(), resp.Header.Get(coscrypto.COSClientSideEncryptionCekAlg), "AES/CTR/NoPadding", "meta data isn't consistent") - assert.Equal(s.T(), resp.Header.Get(coscrypto.COSClientSideEncryptionWrapAlg), "COS/KMS/Crypto", "meta data isn't consistent") + assert.Equal(s.T(), resp.Header.Get(coscrypto.COSClientSideEncryptionCekAlg), coscrypto.AesCtrAlgorithm, "meta data isn't consistent") + assert.Equal(s.T(), resp.Header.Get(coscrypto.COSClientSideEncryptionWrapAlg), coscrypto.CosKmsCryptoWrap, "meta data isn't consistent") assert.Equal(s.T(), resp.Header.Get(coscrypto.COSClientSideEncryptionUnencryptedContentMD5), base64.StdEncoding.EncodeToString(contentMD5), "meta data isn't consistent") fd, err := os.Open(downfile) From 27cd02789e00ecdbb38bf20325ad305e59df7986 Mon Sep 17 00:00:00 2001 From: jojoliang Date: Thu, 29 Apr 2021 17:02:08 +0800 Subject: [PATCH 08/11] update test --- object_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/object_test.go b/object_test.go index 8c7b703..414ff29 100644 --- a/object_test.go +++ b/object_test.go @@ -564,7 +564,6 @@ func TestObjectService_Upload2(t *testing.T) { } } -/* func TestObjectService_Download(t *testing.T) { setup() defer teardown() @@ -637,7 +636,7 @@ func TestObjectService_Download(t *testing.T) { t.Fatalf("Object.Upload returned error: %v", err) } } -*/ + func TestObjectService_DownloadWithCheckPoint(t *testing.T) { setup() defer teardown() From 70b911ba7ef0775d3ca6d3e966cc1c2bea52fd0c Mon Sep 17 00:00:00 2001 From: jojoliang Date: Thu, 29 Apr 2021 19:18:55 +0800 Subject: [PATCH 09/11] update crypto demo --- example/crypto/crypto_sample.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/example/crypto/crypto_sample.go b/example/crypto/crypto_sample.go index e115ecc..925c43f 100644 --- a/example/crypto/crypto_sample.go +++ b/example/crypto/crypto_sample.go @@ -59,8 +59,9 @@ func simple_put_object() { fmt.Println("============== simple_put_object ======================") // 该标识信息唯一确认一个主加密密钥, 解密时，需要传入相同的标识信息 + // KMS加密时，该信息设置成EncryptionContext，最大支持1024字符，如果Encrypt指定了该参数，则在Decrypt 时需要提供同样的参数 materialDesc := make(map[string]string) - materialDesc["desc"] = "" + //materialDesc["desc"] = "material information of your master encrypt key" // 创建KMS客户端 kmsclient, _ := coscrypto.NewKMSClient(c.GetCredential(), "ap-guangzhou") @@ -114,8 +115,9 @@ func simple_put_object_from_file() { fmt.Println("============== simple_put_object_from_file ======================") // 该标识信息唯一确认一个主加密密钥, 解密时，需要传入相同的标识信息 + // KMS加密时，该信息设置成EncryptionContext，最大支持1024字符，如果Encrypt指定了该参数，则在Decrypt 时需要提供同样的参数 materialDesc := make(map[string]string) - materialDesc["desc"] = "" + //materialDesc["desc"] = "material information of your master encrypt key" // 创建KMS客户端 kmsclient, _ := coscrypto.NewKMSClient(c.GetCredential(), "ap-guangzhou") @@ -173,8 +175,9 @@ func multi_put_object() { fmt.Println("============== multi_put_object ======================") // 该标识信息唯一确认一个主加密密钥, 解密时，需要传入相同的标识信息 + // KMS加密时，该信息设置成EncryptionContext，最大支持1024字符，如果Encrypt指定了该参数，则在Decrypt 时需要提供同样的参数 materialDesc := make(map[string]string) - materialDesc["desc"] = "" + //materialDesc["desc"] = "material information of your master encrypt key" // 创建KMS客户端 kmsclient, _ := coscrypto.NewKMSClient(c.GetCredential(), "ap-guangzhou") @@ -244,8 +247,9 @@ func multi_put_object_from_file() { fmt.Println("============== multi_put_object_from_file ======================") // 该标识信息唯一确认一个主加密密钥, 解密时，需要传入相同的标识信息 + // KMS加密时，该信息设置成EncryptionContext，最大支持1024字符，如果Encrypt指定了该参数，则在Decrypt 时需要提供同样的参数 materialDesc := make(map[string]string) - materialDesc["desc"] = "" + //materialDesc["desc"] = "material information of your master encrypt key" // 创建KMS客户端 kmsclient, _ := coscrypto.NewKMSClient(c.GetCredential(), "ap-guangzhou") From 05379cf3101c3f44b59dbbf1bcc14517a8c2f922 Mon Sep 17 00:00:00 2001 From: jojoliang Date: Fri, 14 May 2021 19:54:02 +0800 Subject: [PATCH 10/11] cse-kms done --- cos.go | 2 +- crypto/crypto_object.go | 33 ++++++++++++++++++++++++++++----- crypto/crypto_object_part_test.go | 8 ++++++-- crypto/crypto_object_test.go | 8 ++++++-- crypto/master_kms_cipher.go | 1 + example/crypto/crypto_sample.go | 11 +++++++++-- object.go | 6 ++++++ 7 files changed, 57 insertions(+), 12 deletions(-) diff --git a/cos.go b/cos.go index a09b62f..e15a382 100644 --- a/cos.go +++ b/cos.go @@ -22,7 +22,7 @@ import ( const ( // Version current go sdk version - Version = "0.7.25" + Version = "0.7.26" userAgent = "cos-go-sdk-v5/" + Version contentTypeXML = "application/xml" defaultServiceBaseURL = "http://service.cos.myqcloud.com" diff --git a/crypto/crypto_object.go b/crypto/crypto_object.go index 7c8f1f7..8566faa 100644 --- a/crypto/crypto_object.go +++ b/crypto/crypto_object.go @@ -2,6 +2,7 @@ package coscrypto import ( "context" + "encoding/base64" "fmt" "github.com/tencentyun/cos-go-sdk-v5" "io" @@ -107,7 +108,10 @@ func (s *CryptoObjectService) Get(ctx context.Context, name string, opt *cos.Obj return s.ObjectService.Get(ctx, name, opt, id...) } - envelope := getEnvelopeFromHeader(&meta.Header) + envelope, err := getEnvelopeFromHeader(&meta.Header) + if err != nil { + return nil, err + } if !envelope.IsValid() { return nil, fmt.Errorf("get envelope from header failed, object:%v", name) } @@ -208,20 +212,39 @@ func addCryptoHeaders(header *http.Header, cd *CipherData) { if cd.MatDesc != "" { header.Add(COSClientSideEncryptionMatDesc, cd.MatDesc) } - header.Add(COSClientSideEncryptionKey, string(cd.EncryptedKey)) - header.Add(COSClientSideEncryptionStart, string(cd.EncryptedIV)) + // encrypted key + strEncryptedKey := base64.StdEncoding.EncodeToString(cd.EncryptedKey) + header.Add(COSClientSideEncryptionKey, strEncryptedKey) + + // encrypted iv + strEncryptedIV := base64.StdEncoding.EncodeToString(cd.EncryptedIV) + header.Add(COSClientSideEncryptionStart, strEncryptedIV) + header.Add(COSClientSideEncryptionWrapAlg, cd.WrapAlgorithm) header.Add(COSClientSideEncryptionCekAlg, cd.CEKAlgorithm) } -func getEnvelopeFromHeader(header *http.Header) Envelope { +func getEnvelopeFromHeader(header *http.Header) (Envelope, error) { var envelope Envelope + envelope.CipherKey = header.Get(COSClientSideEncryptionKey) + decodedKey, err := base64.StdEncoding.DecodeString(envelope.CipherKey) + if err != nil { + return envelope, err + } + envelope.CipherKey = string(decodedKey) + envelope.IV = header.Get(COSClientSideEncryptionStart) + decodedIV, err := base64.StdEncoding.DecodeString(envelope.IV) + if err != nil { + return envelope, err + } + envelope.IV = string(decodedIV) + envelope.MatDesc = header.Get(COSClientSideEncryptionMatDesc) envelope.WrapAlg = header.Get(COSClientSideEncryptionWrapAlg) envelope.CEKAlg = header.Get(COSClientSideEncryptionCekAlg) - return envelope + return envelope, nil } func isEncrypted(header *http.Header) bool { diff --git a/crypto/crypto_object_part_test.go b/crypto/crypto_object_part_test.go index 75a05f5..c9af052 100644 --- a/crypto/crypto_object_part_test.go +++ b/crypto/crypto_object_part_test.go @@ -104,10 +104,14 @@ func (s *CosTestSuite) TestMultiUpload_DecryptWithKey() { resp, err = s.CClient.Object.Head(context.Background(), name, nil) assert.Nil(s.T(), err, "HeadObject Failed") cipherKey := resp.Header.Get(coscrypto.COSClientSideEncryptionKey) + cipherKeybs, err := base64.StdEncoding.DecodeString(cipherKey) + assert.Nil(s.T(), err, "base64 Decode Failed") cipherIV := resp.Header.Get(coscrypto.COSClientSideEncryptionStart) - key, err := s.Master.Decrypt([]byte(cipherKey)) + cipherIVbs, err := base64.StdEncoding.DecodeString(cipherIV) + assert.Nil(s.T(), err, "base64 Decode Failed") + key, err := s.Master.Decrypt(cipherKeybs) assert.Nil(s.T(), err, "Master Decrypt Failed") - iv, err := s.Master.Decrypt([]byte(cipherIV)) + iv, err := s.Master.Decrypt(cipherIVbs) assert.Nil(s.T(), err, "Master Decrypt Failed") // 手动解密 diff --git a/crypto/crypto_object_test.go b/crypto/crypto_object_test.go index a662c71..b95c27e 100644 --- a/crypto/crypto_object_test.go +++ b/crypto/crypto_object_test.go @@ -75,10 +75,14 @@ func (s *CosTestSuite) TestPutGetDeleteObject_DecryptWithKey_10MB() { resp, err := s.CClient.Object.Head(context.Background(), name, nil) assert.Nil(s.T(), err, "HeadObject Failed") cipherKey := resp.Header.Get(coscrypto.COSClientSideEncryptionKey) + cipherKeybs, err := base64.StdEncoding.DecodeString(cipherKey) + assert.Nil(s.T(), err, "base64 Decode Failed") cipherIV := resp.Header.Get(coscrypto.COSClientSideEncryptionStart) - key, err := s.Master.Decrypt([]byte(cipherKey)) + cipherIVbs, err := base64.StdEncoding.DecodeString(cipherIV) + assert.Nil(s.T(), err, "base64 Decode Failed") + key, err := s.Master.Decrypt(cipherKeybs) assert.Nil(s.T(), err, "Master Decrypt Failed") - iv, err := s.Master.Decrypt([]byte(cipherIV)) + iv, err := s.Master.Decrypt(cipherIVbs) assert.Nil(s.T(), err, "Master Decrypt Failed") // 正常读取 diff --git a/crypto/master_kms_cipher.go b/crypto/master_kms_cipher.go index 011458e..2a49335 100644 --- a/crypto/master_kms_cipher.go +++ b/crypto/master_kms_cipher.go @@ -64,6 +64,7 @@ func (kc *MasterKMSCipher) Encrypt(plaintext []byte) ([]byte, error) { if err != nil { return nil, err } + // https://cloud.tencent.com/document/product/573/34420 文档有误，返回的结果并没有base64编码 return []byte(*resp.Response.CiphertextBlob), nil } diff --git a/example/crypto/crypto_sample.go b/example/crypto/crypto_sample.go index 925c43f..7ea43c0 100644 --- a/example/crypto/crypto_sample.go +++ b/example/crypto/crypto_sample.go @@ -39,6 +39,13 @@ func log_status(err error) { os.Exit(1) } +func cos_max(x, y int64) int64 { + if x > y { + return x + } + return y +} + func simple_put_object() { u, _ := url.Parse("https://test-1259654469.cos.ap-guangzhou.myqcloud.com") b := &cos.BaseURL{BucketURL: u} @@ -196,7 +203,7 @@ func multi_put_object() { cryptoCtx := coscrypto.CryptoContext{ DataSize: contentLength, // 每个分块需要16字节对齐 - PartSize: (contentLength / 16 / 3) * 16, + PartSize: cos_max(1024*1024, (contentLength/16/3)*16), } v, _, err := client.Object.InitiateMultipartUpload(context.Background(), name, nil, &cryptoCtx) log_status(err) @@ -268,7 +275,7 @@ func multi_put_object_from_file() { cryptoCtx := coscrypto.CryptoContext{ DataSize: contentLength, // 每个分块需要16字节对齐 - PartSize: (contentLength / 16 / 3) * 16, + PartSize: cos_max(1024*1024, (contentLength/16/3)*16), } // 切分数据 _, chunks, _, err := cos.SplitFileIntoChunks(filepath, cryptoCtx.PartSize) diff --git a/object.go b/object.go index 078b92d..5d7fed2 100644 --- a/object.go +++ b/object.go @@ -768,6 +768,9 @@ func SplitFileIntoChunks(filePath string, partSize int64) (int64, []Chunk, int, } var partNum int64 if partSize > 0 { + if partSize < 1024*1024 { + return 0, nil, 0, errors.New("partSize>=1048576 is required") + } partNum = stat.Size() / partSize if partNum >= 10000 { return 0, nil, 0, errors.New("Too many parts, out of 10000") @@ -1066,6 +1069,9 @@ func (s *ObjectService) Upload(ctx context.Context, name string, filepath string func SplitSizeIntoChunks(totalBytes int64, partSize int64) ([]Chunk, int, error) { var partNum int64 if partSize > 0 { + if partSize < 1024*1024 { + return nil, 0, errors.New("partSize>=1048576 is required") + } partNum = totalBytes / partSize if partNum >= 10000 { return nil, 0, errors.New("Too manry parts, out of 10000") From e89b7b088ab69340ef04cd3ab4bb53ad7b4dc40c Mon Sep 17 00:00:00 2001 From: jojoliang Date: Fri, 14 May 2021 20:13:21 +0800 Subject: [PATCH 11/11] update --- crypto/master_kms_cipher.go | 1 - 1 file changed, 1 deletion(-) diff --git a/crypto/master_kms_cipher.go b/crypto/master_kms_cipher.go index 2a49335..011458e 100644 --- a/crypto/master_kms_cipher.go +++ b/crypto/master_kms_cipher.go @@ -64,7 +64,6 @@ func (kc *MasterKMSCipher) Encrypt(plaintext []byte) ([]byte, error) { if err != nil { return nil, err } - // https://cloud.tencent.com/document/product/573/34420 文档有误，返回的结果并没有base64编码 return []byte(*resp.Response.CiphertextBlob), nil }