diff --git a/crypto/aes_ctr.go b/crypto/aes_ctr.go new file mode 100644 index 0000000..1057c2c --- /dev/null +++ b/crypto/aes_ctr.go @@ -0,0 +1,67 @@ +package coscrypto + +import ( + "crypto/aes" + "crypto/cipher" + "io" +) + +type aesCtr struct { + encrypter cipher.Stream + decrypter cipher.Stream +} + +func newAesCtr(cd CipherData) (Cipher, error) { + block, err := aes.NewCipher(cd.Key) + if err != nil { + return nil, err + } + encrypter := cipher.NewCTR(block, cd.IV) + decrypter := cipher.NewCTR(block, cd.IV) + return &aesCtr{encrypter, decrypter}, nil +} + +func (c *aesCtr) Encrypt(src io.Reader) io.Reader { + reader := &ctrEncryptReader{ + encrypter: c.encrypter, + src: src, + } + return reader +} + +type ctrEncryptReader struct { + encrypter cipher.Stream + src io.Reader +} + +func (reader *ctrEncryptReader) Read(data []byte) (int, error) { + plainText := make([]byte, len(data), len(data)) + n, err := reader.src.Read(plainText) + if n > 0 { + plainText = plainText[0:n] + reader.encrypter.XORKeyStream(data, plainText) + } + return n, err +} + +func (c *aesCtr) Decrypt(src io.Reader) io.Reader { + return &ctrDecryptReader{ + decrypter: c.decrypter, + src: src, + } +} + +type ctrDecryptReader struct { + decrypter cipher.Stream + src io.Reader +} + +func (reader *ctrDecryptReader) Read(data []byte) (int, error) { + cryptoText := make([]byte, len(data), len(data)) + n, err := reader.src.Read(cryptoText) + if n > 0 { + cryptoText = cryptoText[0:n] + reader.decrypter.XORKeyStream(data, cryptoText) + } + return n, err +} diff --git a/crypto/aes_ctr_cipher.go b/crypto/aes_ctr_cipher.go new file mode 100644 index 0000000..c140cd7 --- /dev/null +++ b/crypto/aes_ctr_cipher.go @@ -0,0 +1,138 @@ +package coscrypto + +import ( + "io" +) + +const ( + aesKeySize = 32 + ivSize = 16 +) + +type aesCtrCipherBuilder struct { + MasterCipher MasterCipher +} + +type aesCtrCipher struct { + CipherData CipherData + Cipher Cipher +} + +func CreateAesCtrBuilder(cipher MasterCipher) ContentCipherBuilder { + return aesCtrCipherBuilder{MasterCipher: cipher} +} + +func (builder aesCtrCipherBuilder) createCipherData() (CipherData, error) { + var cd CipherData + var err error + err = cd.RandomKeyIv(aesKeySize, ivSize) + if err != nil { + return cd, err + } + + cd.WrapAlgorithm = builder.MasterCipher.GetWrapAlgorithm() + cd.CEKAlgorithm = AesCtrAlgorithm + cd.MatDesc = builder.MasterCipher.GetMatDesc() + + // EncryptedKey + cd.EncryptedKey, err = builder.MasterCipher.Encrypt(cd.Key) + if err != nil { + return cd, err + } + + // EncryptedIV + cd.EncryptedIV, err = builder.MasterCipher.Encrypt(cd.IV) + if err != nil { + return cd, err + } + + return cd, nil +} + +func (builder aesCtrCipherBuilder) contentCipherCD(cd CipherData) (ContentCipher, error) { + cipher, err := newAesCtr(cd) + if err != nil { + return nil, err + } + + return &aesCtrCipher{ + CipherData: cd, + Cipher: cipher, + }, nil +} + +func (builder aesCtrCipherBuilder) ContentCipher() (ContentCipher, error) { + cd, err := builder.createCipherData() + if err != nil { + return nil, err + } + return builder.contentCipherCD(cd) +} + +func (builder aesCtrCipherBuilder) ContentCipherEnv(envelope Envelope) (ContentCipher, error) { + var cd CipherData + cd.EncryptedKey = make([]byte, len(envelope.CipherKey)) + copy(cd.EncryptedKey, []byte(envelope.CipherKey)) + + plainKey, err := builder.MasterCipher.Decrypt([]byte(envelope.CipherKey)) + if err != nil { + return nil, err + } + cd.Key = make([]byte, len(plainKey)) + copy(cd.Key, plainKey) + + cd.EncryptedIV = make([]byte, len(envelope.IV)) + copy(cd.EncryptedIV, []byte(envelope.IV)) + + plainIV, err := builder.MasterCipher.Decrypt([]byte(envelope.IV)) + if err != nil { + return nil, err + } + + cd.IV = make([]byte, len(plainIV)) + copy(cd.IV, plainIV) + + cd.MatDesc = envelope.MatDesc + cd.WrapAlgorithm = envelope.WrapAlg + cd.CEKAlgorithm = envelope.CEKAlg + + return builder.contentCipherCD(cd) +} + +func (builder aesCtrCipherBuilder) GetMatDesc() string { + return builder.MasterCipher.GetMatDesc() +} + +func (cc *aesCtrCipher) EncryptContent(src io.Reader) (io.ReadCloser, error) { + reader := cc.Cipher.Encrypt(src) + return &CryptoEncrypter{Body: src, Encrypter: reader}, nil +} + +func (cc *aesCtrCipher) DecryptContent(src io.Reader) (io.ReadCloser, error) { + reader := cc.Cipher.Decrypt(src) + return &CryptoDecrypter{Body: src, Decrypter: reader}, nil +} + +func (cc *aesCtrCipher) GetCipherData() *CipherData { + return &(cc.CipherData) +} + +func (cc *aesCtrCipher) GetEncryptedLen(plainTextLen int64) int64 { + return plainTextLen +} + +func (cc *aesCtrCipher) GetAlignLen() int { + return len(cc.CipherData.IV) +} + +func (cc *aesCtrCipher) Clone(cd CipherData) (ContentCipher, error) { + cipher, err := newAesCtr(cd) + if err != nil { + return nil, err + } + + return &aesCtrCipher{ + CipherData: cd, + Cipher: cipher, + }, nil +} diff --git a/crypto/aes_ctr_cipher_test.go b/crypto/aes_ctr_cipher_test.go new file mode 100644 index 0000000..f8ebd07 --- /dev/null +++ b/crypto/aes_ctr_cipher_test.go @@ -0,0 +1,107 @@ +package coscrypto_test + +import ( + "bytes" + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "github.com/stretchr/testify/assert" + "github.com/tencentyun/cos-go-sdk-v5/crypto" + "io/ioutil" + math_rand "math/rand" +) + +type EmptyMasterCipher struct{} + +func (mc EmptyMasterCipher) Encrypt(b []byte) ([]byte, error) { + return b, nil +} +func (mc EmptyMasterCipher) Decrypt(b []byte) ([]byte, error) { + return b, nil +} +func (mc EmptyMasterCipher) GetWrapAlgorithm() string { + return "Test/EmptyWrapAlgo" +} +func (mc EmptyMasterCipher) GetMatDesc() string { + return "Empty Desc" +} + +func (s *CosTestSuite) TestCryptoObjectService_EncryptAndDecrypt() { + var masterCipher EmptyMasterCipher + builder := coscrypto.CreateAesCtrBuilder(masterCipher) + + contentCipher, err := builder.ContentCipher() + assert.Nil(s.T(), err, "CryptoObject.CreateAesCtrBuilder Failed") + + dataSize := math_rand.Int63n(1024 * 1024 * 32) + originData := make([]byte, dataSize) + rand.Read(originData) + // 加密 + r1 := bytes.NewReader(originData) + reader1, err := contentCipher.EncryptContent(r1) + assert.Nil(s.T(), err, "CryptoObject.contentCipher.Encrypt Failed") + encryptedData, err := ioutil.ReadAll(reader1) + assert.Nil(s.T(), err, "CryptoObject.Read Failed") + + // 解密 + r2 := bytes.NewReader(encryptedData) + reader2, err := contentCipher.DecryptContent(r2) + decryptedData, err := ioutil.ReadAll(reader2) + assert.Nil(s.T(), err, "CryptoObject.Read Failed") + assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData") +} + +func (s *CosTestSuite) TestCryptoObjectService_Encrypt() { + var masterCipher EmptyMasterCipher + builder := coscrypto.CreateAesCtrBuilder(masterCipher) + + contentCipher, err := builder.ContentCipher() + assert.Nil(s.T(), err, "CryptoObject.CreateAesCtrBuilder Failed") + + dataSize := math_rand.Int63n(1024 * 1024 * 32) + originData := make([]byte, dataSize) + rand.Read(originData) + + // 加密 + r := bytes.NewReader(originData) + reader, err := contentCipher.EncryptContent(r) + assert.Nil(s.T(), err, "CryptoObject.contentCipher.Encrypt Failed") + encryptedData, err := ioutil.ReadAll(reader) + assert.Nil(s.T(), err, "CryptoObject.Read Failed") + + // 直接解密 + cd := contentCipher.GetCipherData() + block, err := aes.NewCipher(cd.Key) + assert.Nil(s.T(), err, "CryptoObject.NewCipher Failed") + decrypter := cipher.NewCTR(block, cd.IV) + decryptedData := make([]byte, len(originData)) + decrypter.XORKeyStream(decryptedData, encryptedData) + assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData") +} + +func (s *CosTestSuite) TestCryptoObjectService_Decrypt() { + var masterCipher EmptyMasterCipher + builder := coscrypto.CreateAesCtrBuilder(masterCipher) + + contentCipher, err := builder.ContentCipher() + assert.Nil(s.T(), err, "CryptoObject.CreateAesCtrBuilder Failed") + dataSize := math_rand.Int63n(1024 * 1024 * 32) + originData := make([]byte, dataSize) + rand.Read(originData) + + // 直接加密 + cd := contentCipher.GetCipherData() + block, err := aes.NewCipher(cd.Key) + assert.Nil(s.T(), err, "CryptoObject.NewCipher Failed") + encrypter := cipher.NewCTR(block, cd.IV) + encryptedData := make([]byte, len(originData)) + encrypter.XORKeyStream(encryptedData, originData) + + // 解密 + r := bytes.NewReader(encryptedData) + reader, err := contentCipher.DecryptContent(r) + assert.Nil(s.T(), err, "CryptoObject.contentCipher.Encrypt Failed") + decryptedData, err := ioutil.ReadAll(reader) + assert.Nil(s.T(), err, "CryptoObject.Read Failed") + assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData") +} diff --git a/crypto/cipher.go b/crypto/cipher.go new file mode 100644 index 0000000..445f3e1 --- /dev/null +++ b/crypto/cipher.go @@ -0,0 +1,69 @@ +package coscrypto + +import ( + "io" +) + +// Cipher is interface for encryption or decryption of an object +type Cipher interface { + Encrypter + Decrypter +} + +// Encrypter is interface with only encrypt method +type Encrypter interface { + Encrypt(io.Reader) io.Reader +} + +// Decrypter is interface with only decrypt method +type Decrypter interface { + Decrypt(io.Reader) io.Reader +} + +// CryptoEncrypter provides close method for Encrypter +type CryptoEncrypter struct { + Body io.Reader + Encrypter io.Reader + isClosed bool +} + +// Close lets the CryptoEncrypter satisfy io.ReadCloser interface +func (rc *CryptoEncrypter) Close() error { + rc.isClosed = true + if closer, ok := rc.Body.(io.ReadCloser); ok { + return closer.Close() + } + return nil +} + +// Read lets the CryptoEncrypter satisfy io.ReadCloser interface +func (rc *CryptoEncrypter) Read(b []byte) (int, error) { + if rc.isClosed { + return 0, io.EOF + } + return rc.Encrypter.Read(b) +} + +// CryptoDecrypter provides close method for Decrypter +type CryptoDecrypter struct { + Body io.Reader + Decrypter io.Reader + isClosed bool +} + +// Close lets the CryptoDecrypter satisfy io.ReadCloser interface +func (rc *CryptoDecrypter) Close() error { + rc.isClosed = true + if closer, ok := rc.Body.(io.ReadCloser); ok { + return closer.Close() + } + return nil +} + +// Read lets the CryptoDecrypter satisfy io.ReadCloser interface +func (rc *CryptoDecrypter) Read(b []byte) (int, error) { + if rc.isClosed { + return 0, io.EOF + } + return rc.Decrypter.Read(b) +} diff --git a/crypto/crypto_object.go b/crypto/crypto_object.go new file mode 100644 index 0000000..903bf6a --- /dev/null +++ b/crypto/crypto_object.go @@ -0,0 +1,228 @@ +package coscrypto + +import ( + "context" + "fmt" + "github.com/tencentyun/cos-go-sdk-v5" + "io" + "net/http" + "os" + "strconv" +) + +type CryptoObjectService struct { + *cos.ObjectService + cryptoClient *CryptoClient +} + +type CryptoClient struct { + *cos.Client + Object *CryptoObjectService + ContentCipherBuilder ContentCipherBuilder + + userAgent string +} + +func NewCryptoClient(client *cos.Client, masterCipher MasterCipher) *CryptoClient { + cc := &CryptoClient{ + Client: client, + Object: &CryptoObjectService{ + client.Object, + nil, + }, + ContentCipherBuilder: CreateAesCtrBuilder(masterCipher), + } + cc.userAgent = cc.Client.UserAgent + "/" + EncryptionUaSuffix + cc.Object.cryptoClient = cc + + return cc +} + +func (s *CryptoObjectService) Put(ctx context.Context, name string, r io.Reader, opt *cos.ObjectPutOptions) (*cos.Response, error) { + cc, err := s.cryptoClient.ContentCipherBuilder.ContentCipher() + if err != nil { + return nil, err + } + reader, err := cc.EncryptContent(r) + if err != nil { + return nil, err + } + opt = cos.CloneObjectPutOptions(opt) + totalBytes, err := cos.GetReaderLen(r) + if err != nil && opt != nil && opt.Listener != nil && opt.ContentLength == 0 { + return nil, err + } + if err == nil { + if opt != nil && opt.ContentLength == 0 { + // 如果未设置Listener, 非bytes.Buffer/bytes.Reader/strings.Reader由用户指定Contength + if opt.Listener != nil || cos.IsLenReader(r) { + opt.ContentLength = totalBytes + } + } + } + if opt.XOptionHeader == nil { + opt.XOptionHeader = &http.Header{} + } + if opt.ContentMD5 != "" { + opt.XOptionHeader.Add(COSClientSideEncryptionUnencryptedContentMD5, opt.ContentMD5) + opt.ContentMD5 = "" + } + if opt.ContentLength != 0 { + opt.XOptionHeader.Add(COSClientSideEncryptionUnencryptedContentLength, strconv.FormatInt(opt.ContentLength, 10)) + opt.ContentLength = cc.GetEncryptedLen(opt.ContentLength) + } + addCryptoHeaders(opt.XOptionHeader, cc.GetCipherData()) + + return s.ObjectService.Put(ctx, name, reader, opt) +} + +func (s *CryptoObjectService) PutFromFile(ctx context.Context, name, filePath string, opt *cos.ObjectPutOptions) (resp *cos.Response, err error) { + nr := 0 + for nr < 3 { + fd, e := os.Open(filePath) + if e != nil { + err = e + return + } + resp, err = s.Put(ctx, name, fd, opt) + if err != nil { + nr++ + fd.Close() + continue + } + fd.Close() + break + } + return +} + +func (s *CryptoObjectService) Get(ctx context.Context, name string, opt *cos.ObjectGetOptions) (*cos.Response, error) { + meta, err := s.ObjectService.Head(ctx, name, nil) + if err != nil { + return meta, err + } + _isEncrypted := isEncrypted(&meta.Header) + if !_isEncrypted { + return s.ObjectService.Get(ctx, name, opt) + } + + envelope := getEnvelopeFromHeader(&meta.Header) + if !envelope.IsValid() { + return nil, fmt.Errorf("get envelope from header failed, object:%v", name) + } + encryptMatDesc := s.cryptoClient.ContentCipherBuilder.GetMatDesc() + if envelope.MatDesc != encryptMatDesc { + return nil, fmt.Errorf("provided master cipher error, want:%v, return:%v, object:%v", encryptMatDesc, envelope.MatDesc, name) + } + + cc, err := s.cryptoClient.ContentCipherBuilder.ContentCipherEnv(envelope) + if err != nil { + return nil, fmt.Errorf("get content cipher from envelope failed: %v, object:%v", err, name) + } + + optRange, err := cos.GetRangeOptions(opt) + if err != nil { + return nil, err + } + discardAlignLen := int64(0) + // Range请求 + if optRange != nil && optRange.HasStart { + // 加密block对齐 + adjustStart := adjustRangeStart(optRange.Start, int64(cc.GetAlignLen())) + discardAlignLen = optRange.Start - adjustStart + if discardAlignLen > 0 { + optRange.Start = adjustStart + opt = cos.CloneObjectGetOptions(opt) + opt.Range = cos.FormatRangeOptions(optRange) + } + + cd := cc.GetCipherData().Clone() + cd.SeekIV(uint64(adjustStart)) + cc, err = cc.Clone(cd) + if err != nil { + return nil, fmt.Errorf("ContentCipher Clone failed:%v, bject:%v", err, name) + } + } + resp, err := s.ObjectService.Get(ctx, name, opt) + if err != nil { + return resp, err + } + resp.Body, err = cc.DecryptContent(resp.Body) + if err != nil { + return resp, err + } + // 抛弃多读取的数据 + if discardAlignLen > 0 { + resp.Body = &cos.DiscardReadCloser{ + RC: resp.Body, + Discard: int(discardAlignLen), + } + } + return resp, err +} + +func (s *CryptoObjectService) GetToFile(ctx context.Context, name, localpath string, opt *cos.ObjectGetOptions) (*cos.Response, error) { + resp, err := s.Get(ctx, name, opt) + if err != nil { + return resp, err + } + defer resp.Body.Close() + + // If file exist, overwrite it + fd, err := os.OpenFile(localpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660) + if err != nil { + return resp, err + } + + _, err = io.Copy(fd, resp.Body) + fd.Close() + if err != nil { + return resp, err + } + + return resp, nil +} + +func (s *CryptoObjectService) MultiUpload(ctx context.Context, name string, filepath string, opt *cos.MultiUploadOptions) (*cos.CompleteMultipartUploadResult, *cos.Response, error) { + return s.Upload(ctx, name, filepath, opt) +} + +func (s *CryptoObjectService) Upload(ctx context.Context, name string, filepath string, opt *cos.MultiUploadOptions) (*cos.CompleteMultipartUploadResult, *cos.Response, error) { + return nil, nil, fmt.Errorf("CryptoObjectService doesn't support Upload Now") +} + +func (s *CryptoObjectService) Download(ctx context.Context, name string, filepath string, opt *cos.MultiDownloadOptions) (*cos.Response, error) { + return nil, fmt.Errorf("CryptoObjectService doesn't support Download Now") +} + +func adjustRangeStart(start int64, alignLen int64) int64 { + return (start / alignLen) * alignLen +} + +func addCryptoHeaders(header *http.Header, cd *CipherData) { + if cd.MatDesc != "" { + header.Add(COSClientSideEncryptionMatDesc, cd.MatDesc) + } + header.Add(COSClientSideEncryptionKey, string(cd.EncryptedKey)) + header.Add(COSClientSideEncryptionStart, string(cd.EncryptedIV)) + header.Add(COSClientSideEncryptionWrapAlg, cd.WrapAlgorithm) + header.Add(COSClientSideEncryptionCekAlg, cd.CEKAlgorithm) +} + +func getEnvelopeFromHeader(header *http.Header) Envelope { + var envelope Envelope + envelope.CipherKey = header.Get(COSClientSideEncryptionKey) + envelope.IV = header.Get(COSClientSideEncryptionStart) + envelope.MatDesc = header.Get(COSClientSideEncryptionMatDesc) + envelope.WrapAlg = header.Get(COSClientSideEncryptionWrapAlg) + envelope.CEKAlg = header.Get(COSClientSideEncryptionCekAlg) + return envelope +} + +func isEncrypted(header *http.Header) bool { + encryptedKey := header.Get(COSClientSideEncryptionKey) + if len(encryptedKey) > 0 { + return true + } + return false +} diff --git a/crypto/crypto_object_part.go b/crypto/crypto_object_part.go new file mode 100644 index 0000000..15561b4 --- /dev/null +++ b/crypto/crypto_object_part.go @@ -0,0 +1,76 @@ +package coscrypto + +import ( + "context" + "fmt" + "github.com/tencentyun/cos-go-sdk-v5" + "io" + "net/http" + "strconv" +) + +type CryptoContext struct { + DataSize int64 + PartSize int64 + ContentCipher ContentCipher +} + +func partSizeIsValid(partSize int64, alignLen int64) bool { + if partSize%alignLen == 0 { + return true + } + return false +} + +func (s *CryptoObjectService) InitiateMultipartUpload(ctx context.Context, name string, opt *cos.InitiateMultipartUploadOptions, cryptoCtx *CryptoContext) (*cos.InitiateMultipartUploadResult, *cos.Response, error) { + contentCipher, err := s.cryptoClient.ContentCipherBuilder.ContentCipher() + if err != nil { + return nil, nil, err + } + if !partSizeIsValid(cryptoCtx.PartSize, int64(contentCipher.GetAlignLen())) { + return nil, nil, fmt.Errorf("PartSize is invalid, it should be %v aligned", contentCipher.GetAlignLen()) + } + // 添加自定义头部 + cryptoCtx.ContentCipher = contentCipher + opt = cos.CloneInitiateMultipartUploadOptions(opt) + if opt.XOptionHeader == nil { + opt.XOptionHeader = &http.Header{} + } + if opt.ContentMD5 != "" { + opt.XOptionHeader.Add(COSClientSideEncryptionUnencryptedContentMD5, opt.ContentMD5) + opt.ContentMD5 = "" + } + opt.XOptionHeader.Add(COSClientSideEncryptionUnencryptedContentLength, strconv.FormatInt(cryptoCtx.DataSize, 10)) + addCryptoHeaders(opt.XOptionHeader, contentCipher.GetCipherData()) + + return s.ObjectService.InitiateMultipartUpload(ctx, name, opt) +} + +func (s *CryptoObjectService) UploadPart(ctx context.Context, name, uploadID string, partNumber int, r io.Reader, opt *cos.ObjectUploadPartOptions, cryptoCtx *CryptoContext) (*cos.Response, error) { + if cryptoCtx.PartSize == 0 { + return nil, fmt.Errorf("CryptoContext's PartSize is zero") + } + opt = cos.CloneObjectUploadPartOptions(opt) + if cryptoCtx.ContentCipher == nil { + return nil, fmt.Errorf("ContentCipher is nil, Please call the InitiateMultipartUpload") + } + totalBytes, err := cos.GetReaderLen(r) + if err == nil { + // 与 go http 保持一致, 非bytes.Buffer/bytes.Reader/strings.Reader需用户指定ContentLength + if opt != nil && opt.ContentLength == 0 && cos.IsLenReader(r) { + opt.ContentLength = totalBytes + } + } + cd := cryptoCtx.ContentCipher.GetCipherData().Clone() + cd.SeekIV(uint64(partNumber-1) * uint64(cryptoCtx.PartSize)) + cc, err := cryptoCtx.ContentCipher.Clone(cd) + opt.ContentLength = cc.GetEncryptedLen(opt.ContentLength) + if err != nil { + return nil, err + } + reader, err := cc.EncryptContent(r) + if err != nil { + return nil, err + } + return s.ObjectService.UploadPart(ctx, name, uploadID, partNumber, reader, opt) +} diff --git a/crypto/crypto_object_part_test.go b/crypto/crypto_object_part_test.go new file mode 100644 index 0000000..a708b2e --- /dev/null +++ b/crypto/crypto_object_part_test.go @@ -0,0 +1,387 @@ +package coscrypto_test + +import ( + "bytes" + "context" + "crypto/aes" + "crypto/cipher" + "crypto/md5" + "crypto/rand" + "encoding/base64" + "fmt" + "github.com/stretchr/testify/assert" + "github.com/tencentyun/cos-go-sdk-v5" + "github.com/tencentyun/cos-go-sdk-v5/crypto" + "io" + "io/ioutil" + math_rand "math/rand" + "net/http" + "net/url" + "os" + "sort" + "sync" + "time" +) + +func (s *CosTestSuite) TestMultiUpload_Normal() { + name := "test/ObjectPut" + time.Now().Format(time.RFC3339) + contentLength := int64(1024*1024*10 + 1) + originData := make([]byte, contentLength) + _, err := rand.Read(originData) + + cryptoCtx := coscrypto.CryptoContext{ + DataSize: contentLength, + PartSize: (contentLength / 16 / 3) * 16, + } + v, _, err := s.CClient.Object.InitiateMultipartUpload(context.Background(), name, nil, &cryptoCtx) + assert.Nil(s.T(), err, "Init Failed") + chunks, _, err := cos.SplitSizeIntoChunks(contentLength, cryptoCtx.PartSize) + assert.Nil(s.T(), err, "Split Failed") + optcom := &cos.CompleteMultipartUploadOptions{} + for _, chunk := range chunks { + opt := &cos.ObjectUploadPartOptions{ + ContentLength: chunk.Size, + } + f := bytes.NewReader(originData[chunk.OffSet : chunk.OffSet+chunk.Size]) + resp, err := s.CClient.Object.UploadPart(context.Background(), name, v.UploadID, chunk.Number, io.LimitReader(f, chunk.Size), opt, &cryptoCtx) + assert.Nil(s.T(), err, "UploadPart failed") + optcom.Parts = append(optcom.Parts, cos.Object{ + PartNumber: chunk.Number, ETag: resp.Header.Get("ETag"), + }) + } + _, _, err = s.CClient.Object.CompleteMultipartUpload(context.Background(), name, v.UploadID, optcom) + assert.Nil(s.T(), err, "Complete Failed") + + resp, err := s.CClient.Object.Get(context.Background(), name, nil) + assert.Nil(s.T(), err, "GetObject Failed") + defer resp.Body.Close() + decryptedData, _ := ioutil.ReadAll(resp.Body) + assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData") + + _, err = s.CClient.Object.Delete(context.Background(), name) + assert.Nil(s.T(), err, "DeleteObject Failed") +} + +func (s *CosTestSuite) TestMultiUpload_DecryptWithKey() { + name := "test/ObjectPut" + time.Now().Format(time.RFC3339) + contentLength := int64(1024*1024*10 + 1) + originData := make([]byte, contentLength) + _, err := rand.Read(originData) + f := bytes.NewReader(originData) + + // 分块上传 + cryptoCtx := coscrypto.CryptoContext{ + DataSize: contentLength, + PartSize: (contentLength / 16 / 3) * 16, + } + v, _, err := s.CClient.Object.InitiateMultipartUpload(context.Background(), name, nil, &cryptoCtx) + assert.Nil(s.T(), err, "Init Failed") + chunks, _, err := cos.SplitSizeIntoChunks(contentLength, cryptoCtx.PartSize) + assert.Nil(s.T(), err, "Split Failed") + optcom := &cos.CompleteMultipartUploadOptions{} + for _, chunk := range chunks { + opt := &cos.ObjectUploadPartOptions{ + ContentLength: chunk.Size, + Listener: &cos.DefaultProgressListener{}, + } + resp, err := s.CClient.Object.UploadPart(context.Background(), name, v.UploadID, chunk.Number, io.LimitReader(f, chunk.Size), opt, &cryptoCtx) + assert.Nil(s.T(), err, "UploadPart failed") + optcom.Parts = append(optcom.Parts, cos.Object{ + PartNumber: chunk.Number, ETag: resp.Header.Get("ETag"), + }) + } + _, _, err = s.CClient.Object.CompleteMultipartUpload(context.Background(), name, v.UploadID, optcom) + assert.Nil(s.T(), err, "Complete Failed") + + // 正常读取 + resp, err := s.Client.Object.Get(context.Background(), name, nil) + assert.Nil(s.T(), err, "GetObject Failed") + defer resp.Body.Close() + encryptedData, _ := ioutil.ReadAll(resp.Body) + assert.NotEqual(s.T(), bytes.Compare(encryptedData, originData), 0, "encryptedData == originData") + + // 获取解密信息 + resp, err = s.CClient.Object.Head(context.Background(), name, nil) + assert.Nil(s.T(), err, "HeadObject Failed") + cipherKey := resp.Header.Get(coscrypto.COSClientSideEncryptionKey) + cipherIV := resp.Header.Get(coscrypto.COSClientSideEncryptionStart) + key, err := s.Master.Decrypt([]byte(cipherKey)) + assert.Nil(s.T(), err, "Master Decrypt Failed") + iv, err := s.Master.Decrypt([]byte(cipherIV)) + assert.Nil(s.T(), err, "Master Decrypt Failed") + + // 手动解密 + block, err := aes.NewCipher(key) + assert.Nil(s.T(), err, "NewCipher Failed") + decrypter := cipher.NewCTR(block, iv) + decryptedData := make([]byte, len(originData)) + decrypter.XORKeyStream(decryptedData, encryptedData) + assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData") + + _, err = s.CClient.Object.Delete(context.Background(), name) + assert.Nil(s.T(), err, "DeleteObject Failed") +} + +func (s *CosTestSuite) TestMultiUpload_PutFromFile() { + name := "test/ObjectPut" + time.Now().Format(time.RFC3339) + filepath := "tmpfile" + time.Now().Format(time.RFC3339) + newfile, err := os.Create(filepath) + assert.Nil(s.T(), err, "Create File Failed") + defer os.Remove(filepath) + + contentLength := int64(1024*1024*10 + 1) + originData := make([]byte, contentLength) + _, err = rand.Read(originData) + newfile.Write(originData) + newfile.Close() + + m := md5.New() + m.Write(originData) + contentMD5 := m.Sum(nil) + cryptoCtx := coscrypto.CryptoContext{ + DataSize: contentLength, + PartSize: (contentLength / 16 / 3) * 16, + } + v, _, err := s.CClient.Object.InitiateMultipartUpload(context.Background(), name, nil, &cryptoCtx) + assert.Nil(s.T(), err, "Init Failed") + _, chunks, _, err := cos.SplitFileIntoChunks(filepath, cryptoCtx.PartSize) + assert.Nil(s.T(), err, "Split Failed") + optcom := &cos.CompleteMultipartUploadOptions{} + var wg sync.WaitGroup + var mtx sync.Mutex + for _, chunk := range chunks { + wg.Add(1) + go func(chk cos.Chunk) { + defer wg.Done() + fd, err := os.Open(filepath) + assert.Nil(s.T(), err, "Open File Failed") + opt := &cos.ObjectUploadPartOptions{ + ContentLength: chk.Size, + } + fd.Seek(chk.OffSet, os.SEEK_SET) + resp, err := s.CClient.Object.UploadPart(context.Background(), name, v.UploadID, chk.Number, io.LimitReader(fd, chk.Size), opt, &cryptoCtx) + assert.Nil(s.T(), err, "UploadPart failed") + mtx.Lock() + optcom.Parts = append(optcom.Parts, cos.Object{ + PartNumber: chk.Number, ETag: resp.Header.Get("ETag"), + }) + mtx.Unlock() + }(chunk) + } + wg.Wait() + sort.Sort(cos.ObjectList(optcom.Parts)) + _, _, err = s.CClient.Object.CompleteMultipartUpload(context.Background(), name, v.UploadID, optcom) + assert.Nil(s.T(), err, "Complete Failed") + + downfile := "downfile" + time.Now().Format(time.RFC3339) + _, err = s.CClient.Object.GetToFile(context.Background(), name, downfile, nil) + assert.Nil(s.T(), err, "GetObject Failed") + + m = md5.New() + fd, err := os.Open(downfile) + assert.Nil(s.T(), err, "Open File Failed") + defer os.Remove(downfile) + defer fd.Close() + io.Copy(m, fd) + downContentMD5 := m.Sum(nil) + assert.Equal(s.T(), bytes.Compare(contentMD5, downContentMD5), 0, "decryptData != originData") + + _, err = s.CClient.Object.Delete(context.Background(), name) + assert.Nil(s.T(), err, "DeleteObject Failed") +} + +func (s *CosTestSuite) TestMultiUpload_GetWithRange() { + name := "test/ObjectPut" + time.Now().Format(time.RFC3339) + filepath := "tmpfile" + time.Now().Format(time.RFC3339) + newfile, err := os.Create(filepath) + assert.Nil(s.T(), err, "Create File Failed") + defer os.Remove(filepath) + + contentLength := int64(1024*1024*10 + 1) + originData := make([]byte, contentLength) + _, err = rand.Read(originData) + newfile.Write(originData) + newfile.Close() + + m := md5.New() + m.Write(originData) + contentMD5 := m.Sum(nil) + cryptoCtx := coscrypto.CryptoContext{ + DataSize: contentLength, + PartSize: (contentLength / 16 / 3) * 16, + } + iniopt := &cos.InitiateMultipartUploadOptions{ + &cos.ACLHeaderOptions{ + XCosACL: "private", + }, + &cos.ObjectPutHeaderOptions{ + ContentMD5: base64.StdEncoding.EncodeToString(contentMD5), + XCosMetaXXX: &http.Header{}, + }, + } + iniopt.XCosMetaXXX.Add("x-cos-meta-isEncrypted", "true") + + v, _, err := s.CClient.Object.InitiateMultipartUpload(context.Background(), name, iniopt, &cryptoCtx) + assert.Nil(s.T(), err, "Init Failed") + _, chunks, _, err := cos.SplitFileIntoChunks(filepath, cryptoCtx.PartSize) + assert.Nil(s.T(), err, "Split Failed") + optcom := &cos.CompleteMultipartUploadOptions{} + var wg sync.WaitGroup + var mtx sync.Mutex + for _, chunk := range chunks { + wg.Add(1) + go func(chk cos.Chunk) { + defer wg.Done() + fd, err := os.Open(filepath) + assert.Nil(s.T(), err, "Open File Failed") + opt := &cos.ObjectUploadPartOptions{ + ContentLength: chk.Size, + } + fd.Seek(chk.OffSet, os.SEEK_SET) + resp, err := s.CClient.Object.UploadPart(context.Background(), name, v.UploadID, chk.Number, io.LimitReader(fd, chk.Size), opt, &cryptoCtx) + assert.Nil(s.T(), err, "UploadPart failed") + mtx.Lock() + optcom.Parts = append(optcom.Parts, cos.Object{ + PartNumber: chk.Number, ETag: resp.Header.Get("ETag"), + }) + mtx.Unlock() + }(chunk) + } + wg.Wait() + sort.Sort(cos.ObjectList(optcom.Parts)) + _, _, err = s.CClient.Object.CompleteMultipartUpload(context.Background(), name, v.UploadID, optcom) + assert.Nil(s.T(), err, "Complete Failed") + + // Range解密读取 + for i := 0; i < 10; i++ { + math_rand.Seed(time.Now().UnixNano()) + rangeStart := math_rand.Int63n(contentLength) + rangeEnd := rangeStart + math_rand.Int63n(contentLength-rangeStart) + if rangeEnd == rangeStart || rangeStart >= contentLength-1 { + continue + } + opt := &cos.ObjectGetOptions{ + Range: fmt.Sprintf("bytes=%v-%v", rangeStart, rangeEnd), + Listener: &cos.DefaultProgressListener{}, + } + resp, err := s.CClient.Object.Get(context.Background(), name, opt) + assert.Nil(s.T(), err, "GetObject Failed") + defer resp.Body.Close() + decryptedData, _ := ioutil.ReadAll(resp.Body) + assert.Equal(s.T(), bytes.Compare(originData[rangeStart:rangeEnd+1], decryptedData), 0, "decryptData != originData") + } + + opt := &cos.ObjectGetOptions{ + Listener: &cos.DefaultProgressListener{}, + } + resp, err := s.CClient.Object.Get(context.Background(), name, opt) + assert.Nil(s.T(), err, "GetObject Failed") + assert.Equal(s.T(), resp.Header.Get("x-cos-meta-isEncrypted"), "true", "meta data isn't consistent") + assert.Equal(s.T(), resp.Header.Get(coscrypto.COSClientSideEncryptionCekAlg), "AES/CTR/NoPadding", "meta data isn't consistent") + assert.Equal(s.T(), resp.Header.Get(coscrypto.COSClientSideEncryptionWrapAlg), "COS/KMS/Crypto", "meta data isn't consistent") + assert.Equal(s.T(), resp.Header.Get(coscrypto.COSClientSideEncryptionUnencryptedContentMD5), base64.StdEncoding.EncodeToString(contentMD5), "meta data isn't consistent") + defer resp.Body.Close() + decryptedData, _ := ioutil.ReadAll(resp.Body) + + assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData") + + _, err = s.CClient.Object.Delete(context.Background(), name) + assert.Nil(s.T(), err, "DeleteObject Failed") +} + +func (s *CosTestSuite) TestMultiUpload_GetWithNewClient() { + name := "test/ObjectPut" + time.Now().Format(time.RFC3339) + contentLength := int64(1024*1024*10 + 1) + originData := make([]byte, contentLength) + _, err := rand.Read(originData) + + cryptoCtx := coscrypto.CryptoContext{ + DataSize: contentLength, + PartSize: (contentLength / 16 / 3) * 16, + } + v, _, err := s.CClient.Object.InitiateMultipartUpload(context.Background(), name, nil, &cryptoCtx) + assert.Nil(s.T(), err, "Init Failed") + chunks, _, err := cos.SplitSizeIntoChunks(contentLength, cryptoCtx.PartSize) + assert.Nil(s.T(), err, "Split Failed") + optcom := &cos.CompleteMultipartUploadOptions{} + for _, chunk := range chunks { + opt := &cos.ObjectUploadPartOptions{ + ContentLength: chunk.Size, + } + f := bytes.NewReader(originData[chunk.OffSet : chunk.OffSet+chunk.Size]) + resp, err := s.CClient.Object.UploadPart(context.Background(), name, v.UploadID, chunk.Number, io.LimitReader(f, chunk.Size), opt, &cryptoCtx) + assert.Nil(s.T(), err, "UploadPart failed") + optcom.Parts = append(optcom.Parts, cos.Object{ + PartNumber: chunk.Number, ETag: resp.Header.Get("ETag"), + }) + } + _, _, err = s.CClient.Object.CompleteMultipartUpload(context.Background(), name, v.UploadID, optcom) + assert.Nil(s.T(), err, "Complete Failed") + + u, _ := url.Parse("https://" + kBucket + ".cos." + kRegion + ".myqcloud.com") + b := &cos.BaseURL{BucketURL: u} + c := cos.NewClient(b, &http.Client{ + Transport: &cos.AuthorizationTransport{ + SecretID: os.Getenv("COS_SECRETID"), + SecretKey: os.Getenv("COS_SECRETKEY"), + }, + }) + { + // 使用不同的MatDesc客户端读取, 期待错误 + material := make(map[string]string) + material["desc"] = "cos crypto suite test 2" + kmsclient, _ := coscrypto.NewKMSClient(c.GetCredential(), kRegion) + master, _ := coscrypto.CreateMasterKMS(kmsclient, os.Getenv("KMSID"), material) + client := coscrypto.NewCryptoClient(c, master) + resp, err := client.Object.Get(context.Background(), name, nil) + assert.Nil(s.T(), resp, "Get Object Failed") + assert.NotNil(s.T(), err, "Get Object Failed") + } + + { + // 使用相同的MatDesc客户端读取, 但KMSID不一样,期待正确,kms解密是不需要KMSID + material := make(map[string]string) + material["desc"] = "cos crypto suite test" + kmsclient, _ := coscrypto.NewKMSClient(c.GetCredential(), kRegion) + master, _ := coscrypto.CreateMasterKMS(kmsclient, "KMSID", material) + client := coscrypto.NewCryptoClient(c, master) + resp, err := client.Object.Get(context.Background(), name, nil) + assert.Nil(s.T(), err, "Get Object Failed") + defer resp.Body.Close() + decryptedData, _ := ioutil.ReadAll(resp.Body) + assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData") + } + + { + // 使用相同的MatDesc客户端读取, 地域不一样,期待错误 + material := make(map[string]string) + material["desc"] = "cos crypto suite test" + diffRegion := "ap-shanghai" + if diffRegion == kRegion { + diffRegion = "ap-guangzhou" + } + kmsclient, _ := coscrypto.NewKMSClient(c.GetCredential(), diffRegion) + master, _ := coscrypto.CreateMasterKMS(kmsclient, "KMSID", material) + client := coscrypto.NewCryptoClient(c, master) + resp, err := client.Object.Get(context.Background(), name, nil) + assert.Nil(s.T(), resp, "Get Object Failed") + assert.NotNil(s.T(), err, "Get Object Failed") + } + + { + // 使用相同的MatDesc和KMSID客户端读取, 期待正确 + material := make(map[string]string) + material["desc"] = "cos crypto suite test" + kmsclient, _ := coscrypto.NewKMSClient(c.GetCredential(), kRegion) + master, _ := coscrypto.CreateMasterKMS(kmsclient, os.Getenv("KMSID"), material) + client := coscrypto.NewCryptoClient(c, master) + resp, err := client.Object.Get(context.Background(), name, nil) + assert.Nil(s.T(), err, "Get Object Failed") + defer resp.Body.Close() + decryptedData, _ := ioutil.ReadAll(resp.Body) + assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData") + } + + _, err = s.CClient.Object.Delete(context.Background(), name) + assert.Nil(s.T(), err, "DeleteObject Failed") +} diff --git a/crypto/crypto_object_test.go b/crypto/crypto_object_test.go new file mode 100644 index 0000000..54bc692 --- /dev/null +++ b/crypto/crypto_object_test.go @@ -0,0 +1,385 @@ +package coscrypto_test + +import ( + "bytes" + "context" + "crypto/aes" + "crypto/cipher" + "crypto/md5" + "crypto/rand" + "encoding/base64" + "fmt" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + "github.com/tencentyun/cos-go-sdk-v5" + "github.com/tencentyun/cos-go-sdk-v5/crypto" + "io" + "io/ioutil" + math_rand "math/rand" + "net/http" + "net/url" + "os" + "testing" + "time" +) + +const ( + kAppid = 1259654469 + kBucket = "cosgosdktest-1259654469" + kRegion = "ap-guangzhou" +) + +type CosTestSuite struct { + suite.Suite + Client *cos.Client + CClient *coscrypto.CryptoClient + Master coscrypto.MasterCipher +} + +func (s *CosTestSuite) SetupSuite() { + u, _ := url.Parse("https://" + kBucket + ".cos." + kRegion + ".myqcloud.com") + b := &cos.BaseURL{BucketURL: u} + s.Client = cos.NewClient(b, &http.Client{ + Transport: &cos.AuthorizationTransport{ + SecretID: os.Getenv("COS_SECRETID"), + SecretKey: os.Getenv("COS_SECRETKEY"), + }, + }) + material := make(map[string]string) + material["desc"] = "cos crypto suite test" + kmsclient, _ := coscrypto.NewKMSClient(s.Client.GetCredential(), kRegion) + s.Master, _ = coscrypto.CreateMasterKMS(kmsclient, os.Getenv("KMSID"), material) + s.CClient = coscrypto.NewCryptoClient(s.Client, s.Master) + opt := &cos.BucketPutOptions{ + XCosACL: "public-read", + } + r, err := s.Client.Bucket.Put(context.Background(), opt) + if err != nil && r != nil && r.StatusCode == 409 { + fmt.Println("BucketAlreadyOwnedByYou") + } else if err != nil { + assert.Nil(s.T(), err, "PutBucket Failed") + } +} + +func (s *CosTestSuite) TestPutGetDeleteObject_DecryptWithKey_10MB() { + name := "test/objectPut" + time.Now().Format(time.RFC3339) + originData := make([]byte, 1024*1024*10+1) + _, err := rand.Read(originData) + f := bytes.NewReader(originData) + + // 加密存储 + _, err = s.CClient.Object.Put(context.Background(), name, f, nil) + assert.Nil(s.T(), err, "PutObject Failed") + + // 获取解密信息 + resp, err := s.CClient.Object.Head(context.Background(), name, nil) + assert.Nil(s.T(), err, "HeadObject Failed") + cipherKey := resp.Header.Get(coscrypto.COSClientSideEncryptionKey) + cipherIV := resp.Header.Get(coscrypto.COSClientSideEncryptionStart) + key, err := s.Master.Decrypt([]byte(cipherKey)) + assert.Nil(s.T(), err, "Master Decrypt Failed") + iv, err := s.Master.Decrypt([]byte(cipherIV)) + assert.Nil(s.T(), err, "Master Decrypt Failed") + + // 正常读取 + resp, err = s.Client.Object.Get(context.Background(), name, nil) + assert.Nil(s.T(), err, "GetObject Failed") + defer resp.Body.Close() + encryptedData, _ := ioutil.ReadAll(resp.Body) + assert.NotEqual(s.T(), bytes.Compare(encryptedData, originData), 0, "encryptedData == originData") + + // 手动解密 + block, err := aes.NewCipher(key) + assert.Nil(s.T(), err, "NewCipher Failed") + decrypter := cipher.NewCTR(block, iv) + decryptedData := make([]byte, len(originData)) + decrypter.XORKeyStream(decryptedData, encryptedData) + assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData") + _, err = s.CClient.Object.Delete(context.Background(), name) + assert.Nil(s.T(), err, "DeleteObject Failed") +} + +func (s *CosTestSuite) TestPutGetDeleteObject_Normal_10MB() { + name := "test/objectPut" + time.Now().Format(time.RFC3339) + originData := make([]byte, 1024*1024*10+1) + _, err := rand.Read(originData) + f := bytes.NewReader(originData) + + // 加密存储 + _, err = s.CClient.Object.Put(context.Background(), name, f, nil) + assert.Nil(s.T(), err, "PutObject Failed") + + // 解密读取 + resp, err := s.CClient.Object.Get(context.Background(), name, nil) + assert.Nil(s.T(), err, "GetObject Failed") + defer resp.Body.Close() + decryptedData, _ := ioutil.ReadAll(resp.Body) + assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData") + + _, err = s.CClient.Object.Delete(context.Background(), name) + assert.Nil(s.T(), err, "DeleteObject Failed") +} + +func (s *CosTestSuite) TestPutGetDeleteObject_ZeroFile() { + name := "test/objectPut" + time.Now().Format(time.RFC3339) + // 加密存储 + _, err := s.CClient.Object.Put(context.Background(), name, bytes.NewReader([]byte("")), nil) + assert.Nil(s.T(), err, "PutObject Failed") + + // 解密读取 + resp, err := s.CClient.Object.Get(context.Background(), name, nil) + assert.Nil(s.T(), err, "GetObject Failed") + defer resp.Body.Close() + decryptedData, _ := ioutil.ReadAll(resp.Body) + assert.Equal(s.T(), bytes.Compare([]byte(""), decryptedData), 0, "decryptData != originData") + + _, err = s.CClient.Object.Delete(context.Background(), name) + assert.Nil(s.T(), err, "DeleteObject Failed") +} + +func (s *CosTestSuite) TestPutGetDeleteObject_WithMetaData() { + name := "test/objectPut" + time.Now().Format(time.RFC3339) + originData := make([]byte, 1024*1024*10+1) + _, err := rand.Read(originData) + f := bytes.NewReader(originData) + + m := md5.New() + m.Write(originData) + contentMD5 := m.Sum(nil) + opt := &cos.ObjectPutOptions{ + &cos.ACLHeaderOptions{ + XCosACL: "private", + }, + &cos.ObjectPutHeaderOptions{ + ContentLength: 1024*1024*10 + 1, + ContentMD5: base64.StdEncoding.EncodeToString(contentMD5), + XCosMetaXXX: &http.Header{}, + }, + } + opt.XCosMetaXXX.Add("x-cos-meta-isEncrypted", "true") + // 加密存储 + _, err = s.CClient.Object.Put(context.Background(), name, f, opt) + assert.Nil(s.T(), err, "PutObject Failed") + + // 解密读取 + resp, err := s.CClient.Object.Get(context.Background(), name, nil) + assert.Nil(s.T(), err, "GetObject Failed") + defer resp.Body.Close() + decryptedData, _ := ioutil.ReadAll(resp.Body) + assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData") + assert.Equal(s.T(), resp.Header.Get("x-cos-meta-isEncrypted"), "true", "meta data isn't consistent") + assert.Equal(s.T(), resp.Header.Get(coscrypto.COSClientSideEncryptionCekAlg), "AES/CTR/NoPadding", "meta data isn't consistent") + assert.Equal(s.T(), resp.Header.Get(coscrypto.COSClientSideEncryptionWrapAlg), "COS/KMS/Crypto", "meta data isn't consistent") + assert.Equal(s.T(), resp.Header.Get(coscrypto.COSClientSideEncryptionUnencryptedContentMD5), base64.StdEncoding.EncodeToString(contentMD5), "meta data isn't consistent") + _, err = s.CClient.Object.Delete(context.Background(), name) + assert.Nil(s.T(), err, "DeleteObject Failed") +} + +func (s *CosTestSuite) TestPutGetDeleteObject_ByFile() { + name := "test/objectPut" + time.Now().Format(time.RFC3339) + filepath := "tmpfile" + time.Now().Format(time.RFC3339) + newfile, err := os.Create(filepath) + assert.Nil(s.T(), err, "Create File Failed") + defer os.Remove(filepath) + + originData := make([]byte, 1024*1024*10+1) + _, err = rand.Read(originData) + newfile.Write(originData) + newfile.Close() + + m := md5.New() + m.Write(originData) + contentMD5 := m.Sum(nil) + opt := &cos.ObjectPutOptions{ + &cos.ACLHeaderOptions{ + XCosACL: "private", + }, + &cos.ObjectPutHeaderOptions{ + ContentLength: 1024*1024*10 + 1, + ContentMD5: base64.StdEncoding.EncodeToString(contentMD5), + XCosMetaXXX: &http.Header{}, + }, + } + opt.XCosMetaXXX.Add("x-cos-meta-isEncrypted", "true") + // 加密存储 + _, err = s.CClient.Object.PutFromFile(context.Background(), name, filepath, opt) + assert.Nil(s.T(), err, "PutFromFile Failed") + + // 解密读取 + downfile := "downfile" + time.Now().Format(time.RFC3339) + resp, err := s.CClient.Object.GetToFile(context.Background(), name, downfile, nil) + assert.Nil(s.T(), err, "GetToFile Failed") + assert.Equal(s.T(), resp.Header.Get("x-cos-meta-isEncrypted"), "true", "meta data isn't consistent") + assert.Equal(s.T(), resp.Header.Get(coscrypto.COSClientSideEncryptionCekAlg), "AES/CTR/NoPadding", "meta data isn't consistent") + assert.Equal(s.T(), resp.Header.Get(coscrypto.COSClientSideEncryptionWrapAlg), "COS/KMS/Crypto", "meta data isn't consistent") + assert.Equal(s.T(), resp.Header.Get(coscrypto.COSClientSideEncryptionUnencryptedContentMD5), base64.StdEncoding.EncodeToString(contentMD5), "meta data isn't consistent") + + fd, err := os.Open(downfile) + assert.Nil(s.T(), err, "Open File Failed") + defer os.Remove(downfile) + defer fd.Close() + m = md5.New() + io.Copy(m, fd) + downContentMD5 := m.Sum(nil) + assert.Equal(s.T(), bytes.Compare(contentMD5, downContentMD5), 0, "decryptData != originData") + _, err = s.CClient.Object.Delete(context.Background(), name) + assert.Nil(s.T(), err, "DeleteObject Failed") +} + +func (s *CosTestSuite) TestPutGetDeleteObject_DecryptWithNewClient_10MB() { + name := "test/objectPut" + time.Now().Format(time.RFC3339) + originData := make([]byte, 1024*1024*10+1) + _, err := rand.Read(originData) + f := bytes.NewReader(originData) + + // 加密存储 + _, err = s.CClient.Object.Put(context.Background(), name, f, nil) + assert.Nil(s.T(), err, "PutObject Failed") + + u, _ := url.Parse("https://" + kBucket + ".cos." + kRegion + ".myqcloud.com") + b := &cos.BaseURL{BucketURL: u} + c := cos.NewClient(b, &http.Client{ + Transport: &cos.AuthorizationTransport{ + SecretID: os.Getenv("COS_SECRETID"), + SecretKey: os.Getenv("COS_SECRETKEY"), + }, + }) + { + // 使用不同的MatDesc客户端读取, 期待错误 + material := make(map[string]string) + material["desc"] = "cos crypto suite test 2" + kmsclient, _ := coscrypto.NewKMSClient(c.GetCredential(), kRegion) + master, _ := coscrypto.CreateMasterKMS(kmsclient, os.Getenv("KMSID"), material) + client := coscrypto.NewCryptoClient(c, master) + resp, err := client.Object.Get(context.Background(), name, nil) + assert.Nil(s.T(), resp, "Get Object Failed") + assert.NotNil(s.T(), err, "Get Object Failed") + } + + { + // 使用相同的MatDesc客户端读取, 但KMSID不一样,期待正确,kms解密是不需要KMSID + material := make(map[string]string) + material["desc"] = "cos crypto suite test" + kmsclient, _ := coscrypto.NewKMSClient(s.Client.GetCredential(), kRegion) + master, _ := coscrypto.CreateMasterKMS(kmsclient, "KMSID", material) + client := coscrypto.NewCryptoClient(c, master) + resp, err := client.Object.Get(context.Background(), name, nil) + assert.Nil(s.T(), err, "Get Object Failed") + defer resp.Body.Close() + decryptedData, _ := ioutil.ReadAll(resp.Body) + assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData") + } + + { + // 使用相同的MatDesc和KMSID客户端读取, 期待正确 + material := make(map[string]string) + material["desc"] = "cos crypto suite test" + kmsclient, _ := coscrypto.NewKMSClient(s.Client.GetCredential(), kRegion) + master, _ := coscrypto.CreateMasterKMS(kmsclient, os.Getenv("KMSID"), material) + client := coscrypto.NewCryptoClient(c, master) + resp, err := client.Object.Get(context.Background(), name, nil) + assert.Nil(s.T(), err, "Get Object Failed") + defer resp.Body.Close() + decryptedData, _ := ioutil.ReadAll(resp.Body) + assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData") + } + + _, err = s.CClient.Object.Delete(context.Background(), name) + assert.Nil(s.T(), err, "DeleteObject Failed") +} + +func (s *CosTestSuite) TestPutGetDeleteObject_RangeGet() { + name := "test/objectPut" + time.Now().Format(time.RFC3339) + contentLength := 1024*1024*10 + 1 + originData := make([]byte, contentLength) + _, err := rand.Read(originData) + f := bytes.NewReader(originData) + + // 加密存储 + _, err = s.CClient.Object.Put(context.Background(), name, f, nil) + assert.Nil(s.T(), err, "PutObject Failed") + + // Range解密读取 + for i := 0; i < 10; i++ { + math_rand.Seed(time.Now().UnixNano()) + rangeStart := math_rand.Intn(contentLength) + rangeEnd := rangeStart + math_rand.Intn(contentLength-rangeStart) + if rangeEnd == rangeStart || rangeStart >= contentLength-1 { + continue + } + opt := &cos.ObjectGetOptions{ + Range: fmt.Sprintf("bytes=%v-%v", rangeStart, rangeEnd), + } + resp, err := s.CClient.Object.Get(context.Background(), name, opt) + assert.Nil(s.T(), err, "GetObject Failed") + defer resp.Body.Close() + decryptedData, _ := ioutil.ReadAll(resp.Body) + assert.Equal(s.T(), bytes.Compare(originData[rangeStart:rangeEnd+1], decryptedData), 0, "decryptData != originData") + } + + // 解密读取 + resp, err := s.CClient.Object.Get(context.Background(), name, nil) + assert.Nil(s.T(), err, "GetObject Failed") + defer resp.Body.Close() + decryptedData, _ := ioutil.ReadAll(resp.Body) + assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData") + + _, err = s.CClient.Object.Delete(context.Background(), name) + assert.Nil(s.T(), err, "DeleteObject Failed") +} + +func (s *CosTestSuite) TestPutGetDeleteObject_WithListenerAndRange() { + name := "test/objectPut" + time.Now().Format(time.RFC3339) + contentLength := 1024*1024*10 + 1 + originData := make([]byte, contentLength) + _, err := rand.Read(originData) + f := bytes.NewReader(originData) + + // 加密存储 + popt := &cos.ObjectPutOptions{ + nil, + &cos.ObjectPutHeaderOptions{ + Listener: &cos.DefaultProgressListener{}, + }, + } + _, err = s.CClient.Object.Put(context.Background(), name, f, popt) + assert.Nil(s.T(), err, "PutObject Failed") + + // Range解密读取 + for i := 0; i < 10; i++ { + math_rand.Seed(time.Now().UnixNano()) + rangeStart := math_rand.Intn(contentLength) + rangeEnd := rangeStart + math_rand.Intn(contentLength-rangeStart) + if rangeEnd == rangeStart || rangeStart >= contentLength-1 { + continue + } + opt := &cos.ObjectGetOptions{ + Range: fmt.Sprintf("bytes=%v-%v", rangeStart, rangeEnd), + Listener: &cos.DefaultProgressListener{}, + } + resp, err := s.CClient.Object.Get(context.Background(), name, opt) + assert.Nil(s.T(), err, "GetObject Failed") + defer resp.Body.Close() + decryptedData, _ := ioutil.ReadAll(resp.Body) + assert.Equal(s.T(), bytes.Compare(originData[rangeStart:rangeEnd+1], decryptedData), 0, "decryptData != originData") + } + // 解密读取 + opt := &cos.ObjectGetOptions{ + Listener: &cos.DefaultProgressListener{}, + } + resp, err := s.CClient.Object.Get(context.Background(), name, opt) + assert.Nil(s.T(), err, "GetObject Failed") + defer resp.Body.Close() + decryptedData, _ := ioutil.ReadAll(resp.Body) + assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData") + + _, err = s.CClient.Object.Delete(context.Background(), name) + assert.Nil(s.T(), err, "DeleteObject Failed") +} + +func TestCosTestSuite(t *testing.T) { + suite.Run(t, new(CosTestSuite)) +} + +func (s *CosTestSuite) TearDownSuite() { +} diff --git a/crypto/crypto_type.go b/crypto/crypto_type.go new file mode 100644 index 0000000..f12557e --- /dev/null +++ b/crypto/crypto_type.go @@ -0,0 +1,141 @@ +package coscrypto + +import ( + "crypto/rand" + "encoding/binary" + "fmt" + "io" + math_rand "math/rand" + "time" +) + +const ( + COSClientSideEncryptionKey string = "x-cos-meta-client-side-encryption-key" + COSClientSideEncryptionStart = "x-cos-meta-client-side-encryption-start" + COSClientSideEncryptionCekAlg = "x-cos-meta-client-side-encryption-cek-alg" + COSClientSideEncryptionWrapAlg = "x-cos-meta-client-side-encryption-wrap-alg" + COSClientSideEncryptionMatDesc = "x-cos-meta-client-side-encryption-matdesc" + COSClientSideEncryptionUnencryptedContentLength = "x-cos-meta-client-side-encryption-unencrypted-content-length" + COSClientSideEncryptionUnencryptedContentMD5 = "x-cos-meta-client-side-encryption-unencrypted-content-md5" + COSClientSideEncryptionDataSize = "x-cos-meta-client-side-encryption-data-size" + COSClientSideEncryptionPartSize = "x-cos-meta-client-side-encryption-part-size" + COSClientUserAgent = "User-Agent" +) + +const ( + CosKmsCryptoWrap = "COS/KMS/Crypto" + AesCtrAlgorithm = "AES/CTR/NoPadding" + EncryptionUaSuffix = "COSEncryptionClient" +) + +type MasterCipher interface { + Encrypt([]byte) ([]byte, error) + Decrypt([]byte) ([]byte, error) + GetWrapAlgorithm() string + GetMatDesc() string +} + +type ContentCipherBuilder interface { + ContentCipher() (ContentCipher, error) + ContentCipherEnv(Envelope) (ContentCipher, error) + GetMatDesc() string +} + +type ContentCipher interface { + EncryptContent(io.Reader) (io.ReadCloser, error) + DecryptContent(io.Reader) (io.ReadCloser, error) + Clone(cd CipherData) (ContentCipher, error) + GetEncryptedLen(int64) int64 + GetCipherData() *CipherData + GetAlignLen() int +} + +type Envelope struct { + IV string + CipherKey string + MatDesc string + WrapAlg string + CEKAlg string + UnencryptedMD5 string + UnencryptedContentLen string +} + +func (el Envelope) IsValid() bool { + return len(el.IV) > 0 && + len(el.CipherKey) > 0 && + len(el.WrapAlg) > 0 && + len(el.CEKAlg) > 0 +} + +func (el Envelope) String() string { + return fmt.Sprintf("IV=%s&CipherKey=%s&WrapAlg=%s&CEKAlg=%s", el.IV, el.CipherKey, el.WrapAlg, el.CEKAlg) +} + +type CipherData struct { + IV []byte + Key []byte + MatDesc string + WrapAlgorithm string + CEKAlgorithm string + EncryptedIV []byte + EncryptedKey []byte +} + +func (cd *CipherData) RandomKeyIv(keyLen int, ivLen int) error { + math_rand.Seed(time.Now().UnixNano()) + + // Key + cd.Key = make([]byte, keyLen) + if _, err := io.ReadFull(rand.Reader, cd.Key); err != nil { + return err + } + + // sizeof uint64 + if ivLen < 8 { + return fmt.Errorf("ivLen:%d less than 8", ivLen) + } + + // IV: | nonce: 8 bytes | Serial number: 8 bytes | + cd.IV = make([]byte, ivLen) + if _, err := io.ReadFull(rand.Reader, cd.IV[0:ivLen-8]); err != nil { + return err + } + + // only use 4 byte,in order not to overflow when SeekIV() + randNumber := math_rand.Uint32() + cd.SetIV(uint64(randNumber)) + return nil +} + +func (cd *CipherData) SetIV(iv uint64) { + ivLen := len(cd.IV) + binary.BigEndian.PutUint64(cd.IV[ivLen-8:], iv) +} + +func (cd *CipherData) GetIV() uint64 { + ivLen := len(cd.IV) + return binary.BigEndian.Uint64(cd.IV[ivLen-8:]) +} + +func (cd *CipherData) SeekIV(startPos uint64) { + cd.SetIV(cd.GetIV() + startPos/uint64(len(cd.IV))) +} + +func (cd *CipherData) Clone() CipherData { + var cloneCd CipherData + cloneCd = *cd + + cloneCd.Key = make([]byte, len(cd.Key)) + copy(cloneCd.Key, cd.Key) + + cloneCd.IV = make([]byte, len(cd.IV)) + copy(cloneCd.IV, cd.IV) + + cloneCd.EncryptedIV = make([]byte, len(cd.EncryptedIV)) + copy(cloneCd.EncryptedIV, cd.EncryptedIV) + + cloneCd.EncryptedKey = make([]byte, len(cd.EncryptedKey)) + copy(cloneCd.EncryptedKey, cd.EncryptedKey) + + return cloneCd +} diff --git a/crypto/master_kms_cipher.go b/crypto/master_kms_cipher.go new file mode 100644 index 0000000..011458e --- /dev/null +++ b/crypto/master_kms_cipher.go @@ -0,0 +1,87 @@ +package coscrypto + +import ( + "encoding/base64" + "encoding/json" + "fmt" + + "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common" + "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/profile" + kms "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/kms/v20190118" + "github.com/tencentyun/cos-go-sdk-v5" +) + +const ( + KMSEndPoint = "kms.tencentcloudapi.com" +) + +type MasterKMSCipher struct { + Client *kms.Client + KmsId string + MatDesc string +} + +func NewKMSClient(cred *cos.Credential, region string) (*kms.Client, error) { + if cred == nil { + fmt.Errorf("credential is nil") + } + credential := common.NewTokenCredential( + cred.SecretID, + cred.SecretKey, + cred.SessionToken, + ) + cpf := profile.NewClientProfile() + cpf.HttpProfile.Endpoint = KMSEndPoint + client, err := kms.NewClient(credential, region, cpf) + return client, err +} + +func CreateMasterKMS(client *kms.Client, kmsId string, desc map[string]string) (MasterCipher, error) { + if kmsId == "" || client == nil { + return nil, fmt.Errorf("KMS ID is empty or kms client is nil") + } + var kmsCipher MasterKMSCipher + var jdesc string + if len(desc) > 0 { + bs, err := json.Marshal(desc) + if err != nil { + return nil, err + } + jdesc = string(bs) + } + kmsCipher.Client = client + kmsCipher.KmsId = kmsId + kmsCipher.MatDesc = jdesc + return &kmsCipher, nil +} + +func (kc *MasterKMSCipher) Encrypt(plaintext []byte) ([]byte, error) { + request := kms.NewEncryptRequest() + request.KeyId = common.StringPtr(kc.KmsId) + request.EncryptionContext = common.StringPtr(kc.MatDesc) + request.Plaintext = common.StringPtr(base64.StdEncoding.EncodeToString(plaintext)) + resp, err := kc.Client.Encrypt(request) + if err != nil { + return nil, err + } + return []byte(*resp.Response.CiphertextBlob), nil +} + +func (kc *MasterKMSCipher) Decrypt(ciphertext []byte) ([]byte, error) { + request := kms.NewDecryptRequest() + request.CiphertextBlob = common.StringPtr(string(ciphertext)) + request.EncryptionContext = common.StringPtr(kc.MatDesc) + resp, err := kc.Client.Decrypt(request) + if err != nil { + return nil, err + } + return base64.StdEncoding.DecodeString(*resp.Response.Plaintext) +} + +func (kc *MasterKMSCipher) GetWrapAlgorithm() string { + return CosKmsCryptoWrap +} + +func (kc *MasterKMSCipher) GetMatDesc() string { + return kc.MatDesc +} diff --git a/crypto/master_kms_cipher_test.go b/crypto/master_kms_cipher_test.go new file mode 100644 index 0000000..2fafe9a --- /dev/null +++ b/crypto/master_kms_cipher_test.go @@ -0,0 +1,91 @@ +package coscrypto_test + +import ( + "bytes" + "crypto/rand" + "encoding/base64" + "encoding/json" + "github.com/stretchr/testify/assert" + "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common" + kms "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/kms/v20190118" + "github.com/tencentyun/cos-go-sdk-v5" + "github.com/tencentyun/cos-go-sdk-v5/crypto" + "os" +) + +func (s *CosTestSuite) TestMasterKmsCipher_TestKmsClient() { + kmsclient, _ := coscrypto.NewKMSClient(&cos.Credential{ + SecretID: os.Getenv("COS_SECRETID"), + SecretKey: os.Getenv("COS_SECRETKEY"), + }, kRegion) + + originData := make([]byte, 1024) + _, err := rand.Read(originData) + + ctx := make(map[string]string) + ctx["desc"] = string(originData[:10]) + bs, _ := json.Marshal(ctx) + ctxJson := string(bs) + enReq := kms.NewEncryptRequest() + enReq.KeyId = common.StringPtr(os.Getenv("KMSID")) + enReq.EncryptionContext = common.StringPtr(ctxJson) + enReq.Plaintext = common.StringPtr(base64.StdEncoding.EncodeToString(originData)) + enResp, err := kmsclient.Encrypt(enReq) + assert.Nil(s.T(), err, "Encrypt Failed") + encryptedData := []byte(*enResp.Response.CiphertextBlob) + + deReq := kms.NewDecryptRequest() + deReq.CiphertextBlob = common.StringPtr(string(encryptedData)) + deReq.EncryptionContext = common.StringPtr(ctxJson) + deResp, err := kmsclient.Decrypt(deReq) + assert.Nil(s.T(), err, "Decrypt Failed") + decryptedData, err := base64.StdEncoding.DecodeString(*deResp.Response.Plaintext) + assert.Nil(s.T(), err, "base64 Decode Failed") + assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "originData != decryptedData") +} + +func (s *CosTestSuite) TestMasterKmsCipher_TestNormal() { + kmsclient, _ := coscrypto.NewKMSClient(&cos.Credential{ + SecretID: os.Getenv("COS_SECRETID"), + SecretKey: os.Getenv("COS_SECRETKEY"), + }, kRegion) + + desc := make(map[string]string) + desc["test"] = "TestMasterKmsCipher_TestNormal" + master, err := coscrypto.CreateMasterKMS(kmsclient, os.Getenv("KMSID"), desc) + assert.Nil(s.T(), err, "CreateMasterKMS Failed") + + originData := make([]byte, 1024) + _, err = rand.Read(originData) + + encryptedData, err := master.Encrypt(originData) + assert.Nil(s.T(), err, "Encrypt Failed") + + decryptedData, err := master.Decrypt(encryptedData) + assert.Nil(s.T(), err, "Decrypt Failed") + + assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "originData != decryptedData") +} + +func (s *CosTestSuite) TestMasterKmsCipher_TestError() { + kmsclient, _ := coscrypto.NewKMSClient(&cos.Credential{ + SecretID: os.Getenv("COS_SECRETID"), + SecretKey: os.Getenv("COS_SECRETKEY"), + }, kRegion) + + desc := make(map[string]string) + desc["test"] = "TestMasterKmsCipher_TestNormal" + master, err := coscrypto.CreateMasterKMS(kmsclient, "ErrorKMSID", desc) + assert.Nil(s.T(), err, "CreateMasterKMS Failed") + + originData := make([]byte, 1024) + _, err = rand.Read(originData) + + encryptedData, err := master.Encrypt(originData) + assert.NotNil(s.T(), err, "Encrypt Failed") + + decryptedData, err := master.Decrypt(encryptedData) + assert.NotNil(s.T(), err, "Decrypt Failed") + + assert.NotEqual(s.T(), bytes.Compare(originData, decryptedData), 0, "originData != decryptedData") +} diff --git a/example/crypto/crypto_sample.go b/example/crypto/crypto_sample.go new file mode 100644 index 0000000..54672ea --- /dev/null +++ b/example/crypto/crypto_sample.go @@ -0,0 +1,305 @@ +package main + +import ( + "bytes" + "context" + "crypto/md5" + "crypto/rand" + "fmt" + "io" + "io/ioutil" + math_rand "math/rand" + "net/http" + "net/url" + "os" + "time" + + "github.com/tencentyun/cos-go-sdk-v5" + "github.com/tencentyun/cos-go-sdk-v5/crypto" + "github.com/tencentyun/cos-go-sdk-v5/debug" +) + +func log_status(err error) { + if err == nil { + return + } + if cos.IsNotFoundError(err) { + // WARN + fmt.Println("WARN: Resource is not existed") + } else if e, ok := cos.IsCOSError(err); ok { + fmt.Printf("ERROR: Code: %v\n", e.Code) + fmt.Printf("ERROR: Message: %v\n", e.Message) + fmt.Printf("ERROR: Resource: %v\n", e.Resource) + fmt.Printf("ERROR: RequestId: %v\n", e.RequestID) + // ERROR + } else { + fmt.Printf("ERROR: %v\n", err) + // ERROR + } + os.Exit(1) +} + +func simple_put_object() { + u, _ := url.Parse("https://test-1259654469.cos.ap-guangzhou.myqcloud.com") + b := &cos.BaseURL{BucketURL: u} + c := cos.NewClient(b, &http.Client{ + Transport: &cos.AuthorizationTransport{ + SecretID: os.Getenv("COS_SECRETID"), + SecretKey: os.Getenv("COS_SECRETKEY"), + Transport: &debug.DebugRequestTransport{ + RequestHeader: true, + RequestBody: false, + ResponseHeader: true, + ResponseBody: false, + }, + }, + }) + // Case1 上传对象 + name := "test/example2" + + fmt.Println("============== simple_put_object ======================") + // 该标识信息唯一确认一个主加密密钥, 解密时,需要传入相同的标识信息 + materialDesc := make(map[string]string) + materialDesc["desc"] = "" + + // 创建KMS客户端 + kmsclient, _ := coscrypto.NewKMSClient(c.GetCredential(), "ap-guangzhou") + // 创建KMS主加密密钥,标识信息和主密钥一一对应 + kmsID := os.Getenv("KMSID") + masterCipher, _ := coscrypto.CreateMasterKMS(kmsclient, kmsID, materialDesc) + // 创建加密客户端 + client := coscrypto.NewCryptoClient(c, masterCipher) + + contentLength := 1024*1024*10 + 1 + originData := make([]byte, contentLength) + _, err := rand.Read(originData) + f := bytes.NewReader(originData) + // 加密上传 + _, err = client.Object.Put(context.Background(), name, f, nil) + log_status(err) + + math_rand.Seed(time.Now().UnixNano()) + rangeStart := math_rand.Intn(contentLength) + rangeEnd := rangeStart + math_rand.Intn(contentLength-rangeStart) + opt := &cos.ObjectGetOptions{ + Range: fmt.Sprintf("bytes=%v-%v", rangeStart, rangeEnd), + } + // 解密下载 + resp, err := client.Object.Get(context.Background(), name, opt) + log_status(err) + defer resp.Body.Close() + decryptedData, _ := ioutil.ReadAll(resp.Body) + if bytes.Compare(decryptedData, originData[rangeStart:rangeEnd+1]) != 0 { + fmt.Println("Error: encryptedData != originData") + } +} + +func simple_put_object_from_file() { + u, _ := url.Parse("https://test-1259654469.cos.ap-guangzhou.myqcloud.com") + b := &cos.BaseURL{BucketURL: u} + c := cos.NewClient(b, &http.Client{ + Transport: &cos.AuthorizationTransport{ + SecretID: os.Getenv("COS_SECRETID"), + SecretKey: os.Getenv("COS_SECRETKEY"), + Transport: &debug.DebugRequestTransport{ + RequestHeader: true, + RequestBody: false, + ResponseHeader: true, + ResponseBody: false, + }, + }, + }) + // Case1 上传对象 + name := "test/example1" + + fmt.Println("============== simple_put_object_from_file ======================") + // 该标识信息唯一确认一个主加密密钥, 解密时,需要传入相同的标识信息 + materialDesc := make(map[string]string) + materialDesc["desc"] = "" + + // 创建KMS客户端 + kmsclient, _ := coscrypto.NewKMSClient(c.GetCredential(), "ap-guangzhou") + // 创建KMS主加密密钥,标识信息和主密钥一一对应 + kmsID := os.Getenv("KMSID") + masterCipher, _ := coscrypto.CreateMasterKMS(kmsclient, kmsID, materialDesc) + // 创建加密客户端 + client := coscrypto.NewCryptoClient(c, masterCipher) + + filepath := "test" + fd, err := os.Open(filepath) + log_status(err) + defer fd.Close() + m := md5.New() + io.Copy(m, fd) + originDataMD5 := m.Sum(nil) + + // 加密上传 + _, err = client.Object.PutFromFile(context.Background(), name, filepath, nil) + log_status(err) + + // 解密下载 + _, err = client.Object.GetToFile(context.Background(), name, "./test.download", nil) + log_status(err) + + fd, err = os.Open("./test.download") + log_status(err) + defer fd.Close() + m = md5.New() + io.Copy(m, fd) + decryptedDataMD5 := m.Sum(nil) + + if bytes.Compare(decryptedDataMD5, originDataMD5) != 0 { + fmt.Println("Error: encryptedData != originData") + } +} + +func multi_put_object() { + u, _ := url.Parse("https://test-1259654469.cos.ap-guangzhou.myqcloud.com") + b := &cos.BaseURL{BucketURL: u} + c := cos.NewClient(b, &http.Client{ + Transport: &cos.AuthorizationTransport{ + SecretID: os.Getenv("COS_SECRETID"), + SecretKey: os.Getenv("COS_SECRETKEY"), + Transport: &debug.DebugRequestTransport{ + RequestHeader: true, + RequestBody: false, + ResponseHeader: true, + ResponseBody: false, + }, + }, + }) + // Case1 上传对象 + name := "test/example1" + + fmt.Println("============== multi_put_object ======================") + // 该标识信息唯一确认一个主加密密钥, 解密时,需要传入相同的标识信息 + materialDesc := make(map[string]string) + materialDesc["desc"] = "" + + // 创建KMS客户端 + kmsclient, _ := coscrypto.NewKMSClient(c.GetCredential(), "ap-guangzhou") + // 创建KMS主加密密钥,标识信息和主密钥一一对应 + kmsID := os.Getenv("KMSID") + masterCipher, _ := coscrypto.CreateMasterKMS(kmsclient, kmsID, materialDesc) + // 创建加密客户端 + client := coscrypto.NewCryptoClient(c, masterCipher) + + contentLength := int64(1024*1024*10 + 1) + originData := make([]byte, contentLength) + _, err := rand.Read(originData) + log_status(err) + + // 分块上传 + cryptoCtx := coscrypto.CryptoContext{ + DataSize: contentLength, + // 每个分块需要16字节对齐 + PartSize: (contentLength / 16 / 3) * 16, + } + v, _, err := client.Object.InitiateMultipartUpload(context.Background(), name, nil, &cryptoCtx) + log_status(err) + // 切分数据 + chunks, _, err := cos.SplitSizeIntoChunks(contentLength, cryptoCtx.PartSize) + log_status(err) + optcom := &cos.CompleteMultipartUploadOptions{} + for _, chunk := range chunks { + opt := &cos.ObjectUploadPartOptions{ + ContentLength: chunk.Size, + } + f := bytes.NewReader(originData[chunk.OffSet : chunk.OffSet+chunk.Size]) + resp, err := client.Object.UploadPart(context.Background(), name, v.UploadID, chunk.Number, f, opt, &cryptoCtx) + log_status(err) + optcom.Parts = append(optcom.Parts, cos.Object{ + PartNumber: chunk.Number, ETag: resp.Header.Get("ETag"), + }) + } + _, _, err = client.Object.CompleteMultipartUpload(context.Background(), name, v.UploadID, optcom) + log_status(err) + + resp, err := client.Object.Get(context.Background(), name, nil) + log_status(err) + defer resp.Body.Close() + decryptedData, _ := ioutil.ReadAll(resp.Body) + if bytes.Compare(decryptedData, originData) != 0 { + fmt.Println("Error: encryptedData != originData") + } +} + +func multi_put_object_from_file() { + u, _ := url.Parse("https://test-1259654469.cos.ap-guangzhou.myqcloud.com") + b := &cos.BaseURL{BucketURL: u} + c := cos.NewClient(b, &http.Client{ + Transport: &cos.AuthorizationTransport{ + SecretID: os.Getenv("COS_SECRETID"), + SecretKey: os.Getenv("COS_SECRETKEY"), + Transport: &debug.DebugRequestTransport{ + RequestHeader: true, + RequestBody: false, + ResponseHeader: true, + ResponseBody: false, + }, + }, + }) + // Case1 上传对象 + name := "test/example1" + + fmt.Println("============== multi_put_object_from_file ======================") + // 该标识信息唯一确认一个主加密密钥, 解密时,需要传入相同的标识信息 + materialDesc := make(map[string]string) + materialDesc["desc"] = "" + + // 创建KMS客户端 + kmsclient, _ := coscrypto.NewKMSClient(c.GetCredential(), "ap-guangzhou") + // 创建KMS主加密密钥,标识信息和主密钥一一对应 + kmsID := os.Getenv("KMSID") + masterCipher, _ := coscrypto.CreateMasterKMS(kmsclient, kmsID, materialDesc) + // 创建加密客户端 + client := coscrypto.NewCryptoClient(c, masterCipher) + + filepath := "test" + stat, err := os.Stat(filepath) + log_status(err) + contentLength := stat.Size() + + // 分块上传 + cryptoCtx := coscrypto.CryptoContext{ + DataSize: contentLength, + // 每个分块需要16字节对齐 + PartSize: (contentLength / 16 / 3) * 16, + } + // 切分数据 + _, chunks, _, err := cos.SplitFileIntoChunks(filepath, cryptoCtx.PartSize) + log_status(err) + + // init mulitupload + v, _, err := client.Object.InitiateMultipartUpload(context.Background(), name, nil, &cryptoCtx) + log_status(err) + + // part upload + optcom := &cos.CompleteMultipartUploadOptions{} + for _, chunk := range chunks { + fd, err := os.Open(filepath) + log_status(err) + opt := &cos.ObjectUploadPartOptions{ + ContentLength: chunk.Size, + } + fd.Seek(chunk.OffSet, os.SEEK_SET) + resp, err := client.Object.UploadPart(context.Background(), name, v.UploadID, chunk.Number, io.LimitReader(fd, chunk.Size), opt, &cryptoCtx) + log_status(err) + optcom.Parts = append(optcom.Parts, cos.Object{ + PartNumber: chunk.Number, ETag: resp.Header.Get("ETag"), + }) + } + // complete upload + _, _, err = client.Object.CompleteMultipartUpload(context.Background(), name, v.UploadID, optcom) + log_status(err) + + _, err = client.Object.GetToFile(context.Background(), name, "test.download", nil) + log_status(err) +} + +func main() { + simple_put_object() + simple_put_object_from_file() + multi_put_object() + multi_put_object_from_file() +}