agin719
4 years ago
committed by
GitHub
GitHub
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 2557 additions and 52 deletions
-
5ci.go
-
24cos.go
-
67crypto/aes_ctr.go
-
138crypto/aes_ctr_cipher.go
-
107crypto/aes_ctr_cipher_test.go
-
69crypto/cipher.go
-
256crypto/crypto_object.go
-
97crypto/crypto_object_part.go
-
391crypto/crypto_object_part_test.go
-
544crypto/crypto_object_test.go
-
141crypto/crypto_type.go
-
87crypto/master_kms_cipher.go
-
91crypto/master_kms_cipher_test.go
-
316example/crypto/crypto_sample.go
-
1go.mod
-
1go.sum
-
103helper.go
-
73helper_test.go
-
46object.go
-
39object_part.go
-
3object_test.go
@ -0,0 +1,67 @@ |
|||
package coscrypto |
|||
|
|||
import ( |
|||
"crypto/aes" |
|||
"crypto/cipher" |
|||
"io" |
|||
) |
|||
|
|||
type aesCtr struct { |
|||
encrypter cipher.Stream |
|||
decrypter cipher.Stream |
|||
} |
|||
|
|||
func newAesCtr(cd CipherData) (Cipher, error) { |
|||
block, err := aes.NewCipher(cd.Key) |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
encrypter := cipher.NewCTR(block, cd.IV) |
|||
decrypter := cipher.NewCTR(block, cd.IV) |
|||
return &aesCtr{encrypter, decrypter}, nil |
|||
} |
|||
|
|||
func (c *aesCtr) Encrypt(src io.Reader) io.Reader { |
|||
reader := &ctrEncryptReader{ |
|||
encrypter: c.encrypter, |
|||
src: src, |
|||
} |
|||
return reader |
|||
} |
|||
|
|||
type ctrEncryptReader struct { |
|||
encrypter cipher.Stream |
|||
src io.Reader |
|||
} |
|||
|
|||
func (reader *ctrEncryptReader) Read(data []byte) (int, error) { |
|||
plainText := make([]byte, len(data), len(data)) |
|||
n, err := reader.src.Read(plainText) |
|||
if n > 0 { |
|||
plainText = plainText[0:n] |
|||
reader.encrypter.XORKeyStream(data, plainText) |
|||
} |
|||
return n, err |
|||
} |
|||
|
|||
func (c *aesCtr) Decrypt(src io.Reader) io.Reader { |
|||
return &ctrDecryptReader{ |
|||
decrypter: c.decrypter, |
|||
src: src, |
|||
} |
|||
} |
|||
|
|||
type ctrDecryptReader struct { |
|||
decrypter cipher.Stream |
|||
src io.Reader |
|||
} |
|||
|
|||
func (reader *ctrDecryptReader) Read(data []byte) (int, error) { |
|||
cryptoText := make([]byte, len(data), len(data)) |
|||
n, err := reader.src.Read(cryptoText) |
|||
if n > 0 { |
|||
cryptoText = cryptoText[0:n] |
|||
reader.decrypter.XORKeyStream(data, cryptoText) |
|||
} |
|||
return n, err |
|||
} |
|||
@ -0,0 +1,138 @@ |
|||
package coscrypto |
|||
|
|||
import ( |
|||
"io" |
|||
) |
|||
|
|||
const ( |
|||
aesKeySize = 32 |
|||
ivSize = 16 |
|||
) |
|||
|
|||
type aesCtrCipherBuilder struct { |
|||
MasterCipher MasterCipher |
|||
} |
|||
|
|||
type aesCtrCipher struct { |
|||
CipherData CipherData |
|||
Cipher Cipher |
|||
} |
|||
|
|||
func CreateAesCtrBuilder(cipher MasterCipher) ContentCipherBuilder { |
|||
return aesCtrCipherBuilder{MasterCipher: cipher} |
|||
} |
|||
|
|||
func (builder aesCtrCipherBuilder) createCipherData() (CipherData, error) { |
|||
var cd CipherData |
|||
var err error |
|||
err = cd.RandomKeyIv(aesKeySize, ivSize) |
|||
if err != nil { |
|||
return cd, err |
|||
} |
|||
|
|||
cd.WrapAlgorithm = builder.MasterCipher.GetWrapAlgorithm() |
|||
cd.CEKAlgorithm = AesCtrAlgorithm |
|||
cd.MatDesc = builder.MasterCipher.GetMatDesc() |
|||
|
|||
// EncryptedKey
|
|||
cd.EncryptedKey, err = builder.MasterCipher.Encrypt(cd.Key) |
|||
if err != nil { |
|||
return cd, err |
|||
} |
|||
|
|||
// EncryptedIV
|
|||
cd.EncryptedIV, err = builder.MasterCipher.Encrypt(cd.IV) |
|||
if err != nil { |
|||
return cd, err |
|||
} |
|||
|
|||
return cd, nil |
|||
} |
|||
|
|||
func (builder aesCtrCipherBuilder) contentCipherCD(cd CipherData) (ContentCipher, error) { |
|||
cipher, err := newAesCtr(cd) |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
|
|||
return &aesCtrCipher{ |
|||
CipherData: cd, |
|||
Cipher: cipher, |
|||
}, nil |
|||
} |
|||
|
|||
func (builder aesCtrCipherBuilder) ContentCipher() (ContentCipher, error) { |
|||
cd, err := builder.createCipherData() |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
return builder.contentCipherCD(cd) |
|||
} |
|||
|
|||
func (builder aesCtrCipherBuilder) ContentCipherEnv(envelope Envelope) (ContentCipher, error) { |
|||
var cd CipherData |
|||
cd.EncryptedKey = make([]byte, len(envelope.CipherKey)) |
|||
copy(cd.EncryptedKey, []byte(envelope.CipherKey)) |
|||
|
|||
plainKey, err := builder.MasterCipher.Decrypt([]byte(envelope.CipherKey)) |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
cd.Key = make([]byte, len(plainKey)) |
|||
copy(cd.Key, plainKey) |
|||
|
|||
cd.EncryptedIV = make([]byte, len(envelope.IV)) |
|||
copy(cd.EncryptedIV, []byte(envelope.IV)) |
|||
|
|||
plainIV, err := builder.MasterCipher.Decrypt([]byte(envelope.IV)) |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
|
|||
cd.IV = make([]byte, len(plainIV)) |
|||
copy(cd.IV, plainIV) |
|||
|
|||
cd.MatDesc = envelope.MatDesc |
|||
cd.WrapAlgorithm = envelope.WrapAlg |
|||
cd.CEKAlgorithm = envelope.CEKAlg |
|||
|
|||
return builder.contentCipherCD(cd) |
|||
} |
|||
|
|||
func (builder aesCtrCipherBuilder) GetMatDesc() string { |
|||
return builder.MasterCipher.GetMatDesc() |
|||
} |
|||
|
|||
func (cc *aesCtrCipher) EncryptContent(src io.Reader) (io.ReadCloser, error) { |
|||
reader := cc.Cipher.Encrypt(src) |
|||
return &CryptoEncrypter{Body: src, Encrypter: reader}, nil |
|||
} |
|||
|
|||
func (cc *aesCtrCipher) DecryptContent(src io.Reader) (io.ReadCloser, error) { |
|||
reader := cc.Cipher.Decrypt(src) |
|||
return &CryptoDecrypter{Body: src, Decrypter: reader}, nil |
|||
} |
|||
|
|||
func (cc *aesCtrCipher) GetCipherData() *CipherData { |
|||
return &(cc.CipherData) |
|||
} |
|||
|
|||
func (cc *aesCtrCipher) GetEncryptedLen(plainTextLen int64) int64 { |
|||
return plainTextLen |
|||
} |
|||
|
|||
func (cc *aesCtrCipher) GetAlignLen() int { |
|||
return len(cc.CipherData.IV) |
|||
} |
|||
|
|||
func (cc *aesCtrCipher) Clone(cd CipherData) (ContentCipher, error) { |
|||
cipher, err := newAesCtr(cd) |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
|
|||
return &aesCtrCipher{ |
|||
CipherData: cd, |
|||
Cipher: cipher, |
|||
}, nil |
|||
} |
|||
@ -0,0 +1,107 @@ |
|||
package coscrypto_test |
|||
|
|||
import ( |
|||
"bytes" |
|||
"crypto/aes" |
|||
"crypto/cipher" |
|||
"crypto/rand" |
|||
"github.com/stretchr/testify/assert" |
|||
"github.com/tencentyun/cos-go-sdk-v5/crypto" |
|||
"io/ioutil" |
|||
math_rand "math/rand" |
|||
) |
|||
|
|||
type EmptyMasterCipher struct{} |
|||
|
|||
func (mc EmptyMasterCipher) Encrypt(b []byte) ([]byte, error) { |
|||
return b, nil |
|||
} |
|||
func (mc EmptyMasterCipher) Decrypt(b []byte) ([]byte, error) { |
|||
return b, nil |
|||
} |
|||
func (mc EmptyMasterCipher) GetWrapAlgorithm() string { |
|||
return "Test/EmptyWrapAlgo" |
|||
} |
|||
func (mc EmptyMasterCipher) GetMatDesc() string { |
|||
return "Empty Desc" |
|||
} |
|||
|
|||
func (s *CosTestSuite) TestCryptoObjectService_EncryptAndDecrypt() { |
|||
var masterCipher EmptyMasterCipher |
|||
builder := coscrypto.CreateAesCtrBuilder(masterCipher) |
|||
|
|||
contentCipher, err := builder.ContentCipher() |
|||
assert.Nil(s.T(), err, "CryptoObject.CreateAesCtrBuilder Failed") |
|||
|
|||
dataSize := math_rand.Int63n(1024 * 1024 * 32) |
|||
originData := make([]byte, dataSize) |
|||
rand.Read(originData) |
|||
// 加密
|
|||
r1 := bytes.NewReader(originData) |
|||
reader1, err := contentCipher.EncryptContent(r1) |
|||
assert.Nil(s.T(), err, "CryptoObject.contentCipher.Encrypt Failed") |
|||
encryptedData, err := ioutil.ReadAll(reader1) |
|||
assert.Nil(s.T(), err, "CryptoObject.Read Failed") |
|||
|
|||
// 解密
|
|||
r2 := bytes.NewReader(encryptedData) |
|||
reader2, err := contentCipher.DecryptContent(r2) |
|||
decryptedData, err := ioutil.ReadAll(reader2) |
|||
assert.Nil(s.T(), err, "CryptoObject.Read Failed") |
|||
assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData") |
|||
} |
|||
|
|||
func (s *CosTestSuite) TestCryptoObjectService_Encrypt() { |
|||
var masterCipher EmptyMasterCipher |
|||
builder := coscrypto.CreateAesCtrBuilder(masterCipher) |
|||
|
|||
contentCipher, err := builder.ContentCipher() |
|||
assert.Nil(s.T(), err, "CryptoObject.CreateAesCtrBuilder Failed") |
|||
|
|||
dataSize := math_rand.Int63n(1024 * 1024 * 32) |
|||
originData := make([]byte, dataSize) |
|||
rand.Read(originData) |
|||
|
|||
// 加密
|
|||
r := bytes.NewReader(originData) |
|||
reader, err := contentCipher.EncryptContent(r) |
|||
assert.Nil(s.T(), err, "CryptoObject.contentCipher.Encrypt Failed") |
|||
encryptedData, err := ioutil.ReadAll(reader) |
|||
assert.Nil(s.T(), err, "CryptoObject.Read Failed") |
|||
|
|||
// 直接解密
|
|||
cd := contentCipher.GetCipherData() |
|||
block, err := aes.NewCipher(cd.Key) |
|||
assert.Nil(s.T(), err, "CryptoObject.NewCipher Failed") |
|||
decrypter := cipher.NewCTR(block, cd.IV) |
|||
decryptedData := make([]byte, len(originData)) |
|||
decrypter.XORKeyStream(decryptedData, encryptedData) |
|||
assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData") |
|||
} |
|||
|
|||
func (s *CosTestSuite) TestCryptoObjectService_Decrypt() { |
|||
var masterCipher EmptyMasterCipher |
|||
builder := coscrypto.CreateAesCtrBuilder(masterCipher) |
|||
|
|||
contentCipher, err := builder.ContentCipher() |
|||
assert.Nil(s.T(), err, "CryptoObject.CreateAesCtrBuilder Failed") |
|||
dataSize := math_rand.Int63n(1024 * 1024 * 32) |
|||
originData := make([]byte, dataSize) |
|||
rand.Read(originData) |
|||
|
|||
// 直接加密
|
|||
cd := contentCipher.GetCipherData() |
|||
block, err := aes.NewCipher(cd.Key) |
|||
assert.Nil(s.T(), err, "CryptoObject.NewCipher Failed") |
|||
encrypter := cipher.NewCTR(block, cd.IV) |
|||
encryptedData := make([]byte, len(originData)) |
|||
encrypter.XORKeyStream(encryptedData, originData) |
|||
|
|||
// 解密
|
|||
r := bytes.NewReader(encryptedData) |
|||
reader, err := contentCipher.DecryptContent(r) |
|||
assert.Nil(s.T(), err, "CryptoObject.contentCipher.Encrypt Failed") |
|||
decryptedData, err := ioutil.ReadAll(reader) |
|||
assert.Nil(s.T(), err, "CryptoObject.Read Failed") |
|||
assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData") |
|||
} |
|||
@ -0,0 +1,69 @@ |
|||
package coscrypto |
|||
|
|||
import ( |
|||
"io" |
|||
) |
|||
|
|||
// Cipher is interface for encryption or decryption of an object
|
|||
type Cipher interface { |
|||
Encrypter |
|||
Decrypter |
|||
} |
|||
|
|||
// Encrypter is interface with only encrypt method
|
|||
type Encrypter interface { |
|||
Encrypt(io.Reader) io.Reader |
|||
} |
|||
|
|||
// Decrypter is interface with only decrypt method
|
|||
type Decrypter interface { |
|||
Decrypt(io.Reader) io.Reader |
|||
} |
|||
|
|||
// CryptoEncrypter provides close method for Encrypter
|
|||
type CryptoEncrypter struct { |
|||
Body io.Reader |
|||
Encrypter io.Reader |
|||
isClosed bool |
|||
} |
|||
|
|||
// Close lets the CryptoEncrypter satisfy io.ReadCloser interface
|
|||
func (rc *CryptoEncrypter) Close() error { |
|||
rc.isClosed = true |
|||
if closer, ok := rc.Body.(io.ReadCloser); ok { |
|||
return closer.Close() |
|||
} |
|||
return nil |
|||
} |
|||
|
|||
// Read lets the CryptoEncrypter satisfy io.ReadCloser interface
|
|||
func (rc *CryptoEncrypter) Read(b []byte) (int, error) { |
|||
if rc.isClosed { |
|||
return 0, io.EOF |
|||
} |
|||
return rc.Encrypter.Read(b) |
|||
} |
|||
|
|||
// CryptoDecrypter provides close method for Decrypter
|
|||
type CryptoDecrypter struct { |
|||
Body io.Reader |
|||
Decrypter io.Reader |
|||
isClosed bool |
|||
} |
|||
|
|||
// Close lets the CryptoDecrypter satisfy io.ReadCloser interface
|
|||
func (rc *CryptoDecrypter) Close() error { |
|||
rc.isClosed = true |
|||
if closer, ok := rc.Body.(io.ReadCloser); ok { |
|||
return closer.Close() |
|||
} |
|||
return nil |
|||
} |
|||
|
|||
// Read lets the CryptoDecrypter satisfy io.ReadCloser interface
|
|||
func (rc *CryptoDecrypter) Read(b []byte) (int, error) { |
|||
if rc.isClosed { |
|||
return 0, io.EOF |
|||
} |
|||
return rc.Decrypter.Read(b) |
|||
} |
|||
@ -0,0 +1,256 @@ |
|||
package coscrypto |
|||
|
|||
import ( |
|||
"context" |
|||
"encoding/base64" |
|||
"fmt" |
|||
"github.com/tencentyun/cos-go-sdk-v5" |
|||
"io" |
|||
"net/http" |
|||
"os" |
|||
"strconv" |
|||
) |
|||
|
|||
type CryptoObjectService struct { |
|||
*cos.ObjectService |
|||
cryptoClient *CryptoClient |
|||
} |
|||
|
|||
type CryptoClient struct { |
|||
*cos.Client |
|||
Object *CryptoObjectService |
|||
ContentCipherBuilder ContentCipherBuilder |
|||
|
|||
userAgent string |
|||
} |
|||
|
|||
func NewCryptoClient(client *cos.Client, masterCipher MasterCipher) *CryptoClient { |
|||
cc := &CryptoClient{ |
|||
Client: client, |
|||
Object: &CryptoObjectService{ |
|||
client.Object, |
|||
nil, |
|||
}, |
|||
ContentCipherBuilder: CreateAesCtrBuilder(masterCipher), |
|||
} |
|||
cc.userAgent = cc.Client.UserAgent + "/" + EncryptionUaSuffix |
|||
cc.Object.cryptoClient = cc |
|||
|
|||
return cc |
|||
} |
|||
|
|||
func (s *CryptoObjectService) Put(ctx context.Context, name string, r io.Reader, opt *cos.ObjectPutOptions) (*cos.Response, error) { |
|||
cc, err := s.cryptoClient.ContentCipherBuilder.ContentCipher() |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
reader, err := cc.EncryptContent(r) |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
opt = cos.CloneObjectPutOptions(opt) |
|||
totalBytes, err := cos.GetReaderLen(r) |
|||
if err != nil && opt != nil && opt.Listener != nil && opt.ContentLength == 0 { |
|||
return nil, err |
|||
} |
|||
if err == nil { |
|||
if opt != nil && opt.ContentLength == 0 { |
|||
// 如果未设置Listener, 非bytes.Buffer/bytes.Reader/strings.Reader由用户指定Contength
|
|||
if opt.Listener != nil || cos.IsLenReader(r) { |
|||
opt.ContentLength = totalBytes |
|||
} |
|||
} |
|||
} |
|||
if opt.XOptionHeader == nil { |
|||
opt.XOptionHeader = &http.Header{} |
|||
} |
|||
if opt.ContentMD5 != "" { |
|||
opt.XOptionHeader.Add(COSClientSideEncryptionUnencryptedContentMD5, opt.ContentMD5) |
|||
opt.ContentMD5 = "" |
|||
} |
|||
if opt.ContentLength != 0 { |
|||
opt.XOptionHeader.Add(COSClientSideEncryptionUnencryptedContentLength, strconv.FormatInt(opt.ContentLength, 10)) |
|||
opt.ContentLength = cc.GetEncryptedLen(opt.ContentLength) |
|||
} |
|||
opt.XOptionHeader.Add(UserAgent, s.cryptoClient.userAgent) |
|||
addCryptoHeaders(opt.XOptionHeader, cc.GetCipherData()) |
|||
|
|||
return s.ObjectService.Put(ctx, name, reader, opt) |
|||
} |
|||
|
|||
func (s *CryptoObjectService) PutFromFile(ctx context.Context, name, filePath string, opt *cos.ObjectPutOptions) (resp *cos.Response, err error) { |
|||
nr := 0 |
|||
for nr < 3 { |
|||
fd, e := os.Open(filePath) |
|||
if e != nil { |
|||
err = e |
|||
return |
|||
} |
|||
resp, err = s.Put(ctx, name, fd, opt) |
|||
if err != nil { |
|||
nr++ |
|||
fd.Close() |
|||
continue |
|||
} |
|||
fd.Close() |
|||
break |
|||
} |
|||
return |
|||
} |
|||
|
|||
func (s *CryptoObjectService) Get(ctx context.Context, name string, opt *cos.ObjectGetOptions, id ...string) (*cos.Response, error) { |
|||
meta, err := s.ObjectService.Head(ctx, name, nil, id...) |
|||
if err != nil { |
|||
return meta, err |
|||
} |
|||
_isEncrypted := isEncrypted(&meta.Header) |
|||
if !_isEncrypted { |
|||
return s.ObjectService.Get(ctx, name, opt, id...) |
|||
} |
|||
|
|||
envelope, err := getEnvelopeFromHeader(&meta.Header) |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
if !envelope.IsValid() { |
|||
return nil, fmt.Errorf("get envelope from header failed, object:%v", name) |
|||
} |
|||
encryptMatDesc := s.cryptoClient.ContentCipherBuilder.GetMatDesc() |
|||
if envelope.MatDesc != encryptMatDesc { |
|||
return nil, fmt.Errorf("provided master cipher error, want:%v, return:%v, object:%v", encryptMatDesc, envelope.MatDesc, name) |
|||
} |
|||
|
|||
cc, err := s.cryptoClient.ContentCipherBuilder.ContentCipherEnv(envelope) |
|||
if err != nil { |
|||
return nil, fmt.Errorf("get content cipher from envelope failed: %v, object:%v", err, name) |
|||
} |
|||
|
|||
opt = cos.CloneObjectGetOptions(opt) |
|||
if opt.XOptionHeader == nil { |
|||
opt.XOptionHeader = &http.Header{} |
|||
} |
|||
optRange, err := cos.GetRangeOptions(opt) |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
discardAlignLen := int64(0) |
|||
// Range请求
|
|||
if optRange != nil && optRange.HasStart { |
|||
// 加密block对齐
|
|||
adjustStart := adjustRangeStart(optRange.Start, int64(cc.GetAlignLen())) |
|||
discardAlignLen = optRange.Start - adjustStart |
|||
if discardAlignLen > 0 { |
|||
optRange.Start = adjustStart |
|||
opt.Range = cos.FormatRangeOptions(optRange) |
|||
} |
|||
|
|||
cd := cc.GetCipherData().Clone() |
|||
cd.SeekIV(uint64(adjustStart)) |
|||
cc, err = cc.Clone(cd) |
|||
if err != nil { |
|||
return nil, fmt.Errorf("ContentCipher Clone failed:%v, bject:%v", err, name) |
|||
} |
|||
} |
|||
opt.XOptionHeader.Add(UserAgent, s.cryptoClient.userAgent) |
|||
resp, err := s.ObjectService.Get(ctx, name, opt, id...) |
|||
if err != nil { |
|||
return resp, err |
|||
} |
|||
resp.Body, err = cc.DecryptContent(resp.Body) |
|||
if err != nil { |
|||
return resp, err |
|||
} |
|||
// 抛弃多读取的数据
|
|||
if discardAlignLen > 0 { |
|||
resp.Body = &cos.DiscardReadCloser{ |
|||
RC: resp.Body, |
|||
Discard: int(discardAlignLen), |
|||
} |
|||
} |
|||
return resp, err |
|||
} |
|||
|
|||
func (s *CryptoObjectService) GetToFile(ctx context.Context, name, localpath string, opt *cos.ObjectGetOptions, id ...string) (*cos.Response, error) { |
|||
resp, err := s.Get(ctx, name, opt, id...) |
|||
if err != nil { |
|||
return resp, err |
|||
} |
|||
defer resp.Body.Close() |
|||
|
|||
// If file exist, overwrite it
|
|||
fd, err := os.OpenFile(localpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660) |
|||
if err != nil { |
|||
return resp, err |
|||
} |
|||
|
|||
_, err = io.Copy(fd, resp.Body) |
|||
fd.Close() |
|||
if err != nil { |
|||
return resp, err |
|||
} |
|||
|
|||
return resp, nil |
|||
} |
|||
|
|||
func (s *CryptoObjectService) MultiUpload(ctx context.Context, name string, filepath string, opt *cos.MultiUploadOptions) (*cos.CompleteMultipartUploadResult, *cos.Response, error) { |
|||
return s.Upload(ctx, name, filepath, opt) |
|||
} |
|||
|
|||
func (s *CryptoObjectService) Upload(ctx context.Context, name string, filepath string, opt *cos.MultiUploadOptions) (*cos.CompleteMultipartUploadResult, *cos.Response, error) { |
|||
return nil, nil, fmt.Errorf("CryptoObjectService doesn't support Upload Now") |
|||
} |
|||
|
|||
func (s *CryptoObjectService) Download(ctx context.Context, name string, filepath string, opt *cos.MultiDownloadOptions) (*cos.Response, error) { |
|||
return nil, fmt.Errorf("CryptoObjectService doesn't support Download Now") |
|||
} |
|||
|
|||
func adjustRangeStart(start int64, alignLen int64) int64 { |
|||
return (start / alignLen) * alignLen |
|||
} |
|||
|
|||
func addCryptoHeaders(header *http.Header, cd *CipherData) { |
|||
if cd.MatDesc != "" { |
|||
header.Add(COSClientSideEncryptionMatDesc, cd.MatDesc) |
|||
} |
|||
// encrypted key
|
|||
strEncryptedKey := base64.StdEncoding.EncodeToString(cd.EncryptedKey) |
|||
header.Add(COSClientSideEncryptionKey, strEncryptedKey) |
|||
|
|||
// encrypted iv
|
|||
strEncryptedIV := base64.StdEncoding.EncodeToString(cd.EncryptedIV) |
|||
header.Add(COSClientSideEncryptionStart, strEncryptedIV) |
|||
|
|||
header.Add(COSClientSideEncryptionWrapAlg, cd.WrapAlgorithm) |
|||
header.Add(COSClientSideEncryptionCekAlg, cd.CEKAlgorithm) |
|||
} |
|||
|
|||
func getEnvelopeFromHeader(header *http.Header) (Envelope, error) { |
|||
var envelope Envelope |
|||
|
|||
envelope.CipherKey = header.Get(COSClientSideEncryptionKey) |
|||
decodedKey, err := base64.StdEncoding.DecodeString(envelope.CipherKey) |
|||
if err != nil { |
|||
return envelope, err |
|||
} |
|||
envelope.CipherKey = string(decodedKey) |
|||
|
|||
envelope.IV = header.Get(COSClientSideEncryptionStart) |
|||
decodedIV, err := base64.StdEncoding.DecodeString(envelope.IV) |
|||
if err != nil { |
|||
return envelope, err |
|||
} |
|||
envelope.IV = string(decodedIV) |
|||
|
|||
envelope.MatDesc = header.Get(COSClientSideEncryptionMatDesc) |
|||
envelope.WrapAlg = header.Get(COSClientSideEncryptionWrapAlg) |
|||
envelope.CEKAlg = header.Get(COSClientSideEncryptionCekAlg) |
|||
return envelope, nil |
|||
} |
|||
|
|||
func isEncrypted(header *http.Header) bool { |
|||
encryptedKey := header.Get(COSClientSideEncryptionKey) |
|||
if len(encryptedKey) > 0 { |
|||
return true |
|||
} |
|||
return false |
|||
} |
|||
@ -0,0 +1,97 @@ |
|||
package coscrypto |
|||
|
|||
import ( |
|||
"context" |
|||
"fmt" |
|||
"github.com/tencentyun/cos-go-sdk-v5" |
|||
"io" |
|||
"net/http" |
|||
"strconv" |
|||
) |
|||
|
|||
type CryptoContext struct { |
|||
DataSize int64 |
|||
PartSize int64 |
|||
ContentCipher ContentCipher |
|||
} |
|||
|
|||
func partSizeIsValid(partSize int64, alignLen int64) bool { |
|||
if partSize%alignLen == 0 { |
|||
return true |
|||
} |
|||
return false |
|||
} |
|||
|
|||
func (s *CryptoObjectService) InitiateMultipartUpload(ctx context.Context, name string, opt *cos.InitiateMultipartUploadOptions, cryptoCtx *CryptoContext) (*cos.InitiateMultipartUploadResult, *cos.Response, error) { |
|||
contentCipher, err := s.cryptoClient.ContentCipherBuilder.ContentCipher() |
|||
if err != nil { |
|||
return nil, nil, err |
|||
} |
|||
if !partSizeIsValid(cryptoCtx.PartSize, int64(contentCipher.GetAlignLen())) { |
|||
return nil, nil, fmt.Errorf("PartSize is invalid, it should be %v aligned", contentCipher.GetAlignLen()) |
|||
} |
|||
// 添加自定义头部
|
|||
cryptoCtx.ContentCipher = contentCipher |
|||
opt = cos.CloneInitiateMultipartUploadOptions(opt) |
|||
if opt.XOptionHeader == nil { |
|||
opt.XOptionHeader = &http.Header{} |
|||
} |
|||
if opt.ContentMD5 != "" { |
|||
opt.XOptionHeader.Add(COSClientSideEncryptionUnencryptedContentMD5, opt.ContentMD5) |
|||
opt.ContentMD5 = "" |
|||
} |
|||
if cryptoCtx.DataSize > 0 { |
|||
opt.XOptionHeader.Add(COSClientSideEncryptionDataSize, strconv.FormatInt(cryptoCtx.DataSize, 10)) |
|||
} |
|||
opt.XOptionHeader.Add(COSClientSideEncryptionPartSize, strconv.FormatInt(cryptoCtx.PartSize, 10)) |
|||
opt.XOptionHeader.Add(UserAgent, s.cryptoClient.userAgent) |
|||
addCryptoHeaders(opt.XOptionHeader, contentCipher.GetCipherData()) |
|||
|
|||
return s.ObjectService.InitiateMultipartUpload(ctx, name, opt) |
|||
} |
|||
|
|||
func (s *CryptoObjectService) UploadPart(ctx context.Context, name, uploadID string, partNumber int, r io.Reader, opt *cos.ObjectUploadPartOptions, cryptoCtx *CryptoContext) (*cos.Response, error) { |
|||
if cryptoCtx.PartSize == 0 { |
|||
return nil, fmt.Errorf("CryptoContext's PartSize is zero") |
|||
} |
|||
opt = cos.CloneObjectUploadPartOptions(opt) |
|||
if opt.XOptionHeader == nil { |
|||
opt.XOptionHeader = &http.Header{} |
|||
} |
|||
opt.XOptionHeader.Add(UserAgent, s.cryptoClient.userAgent) |
|||
if cryptoCtx.ContentCipher == nil { |
|||
return nil, fmt.Errorf("ContentCipher is nil, Please call the InitiateMultipartUpload") |
|||
} |
|||
totalBytes, err := cos.GetReaderLen(r) |
|||
if err == nil { |
|||
// 与 go http 保持一致, 非bytes.Buffer/bytes.Reader/strings.Reader需用户指定ContentLength
|
|||
if opt != nil && opt.ContentLength == 0 && cos.IsLenReader(r) { |
|||
opt.ContentLength = totalBytes |
|||
} |
|||
} |
|||
cd := cryptoCtx.ContentCipher.GetCipherData().Clone() |
|||
cd.SeekIV(uint64(partNumber-1) * uint64(cryptoCtx.PartSize)) |
|||
cc, err := cryptoCtx.ContentCipher.Clone(cd) |
|||
opt.ContentLength = cc.GetEncryptedLen(opt.ContentLength) |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
reader, err := cc.EncryptContent(r) |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
return s.ObjectService.UploadPart(ctx, name, uploadID, partNumber, reader, opt) |
|||
} |
|||
|
|||
func (s *CryptoObjectService) CompleteMultipartUpload(ctx context.Context, name, uploadID string, opt *cos.CompleteMultipartUploadOptions) (*cos.CompleteMultipartUploadResult, *cos.Response, error) { |
|||
opt = cos.CloneCompleteMultipartUploadOptions(opt) |
|||
if opt.XOptionHeader == nil { |
|||
opt.XOptionHeader = &http.Header{} |
|||
} |
|||
opt.XOptionHeader.Add(UserAgent, s.cryptoClient.userAgent) |
|||
return s.ObjectService.CompleteMultipartUpload(ctx, name, uploadID, opt) |
|||
} |
|||
|
|||
func (s *CryptoObjectService) CopyPart(ctx context.Context, name, uploadID string, partNumber int, sourceURL string, opt *cos.ObjectCopyPartOptions) (*cos.CopyPartResult, *cos.Response, error) { |
|||
return nil, nil, fmt.Errorf("CryptoObjectService doesn't support CopyPart") |
|||
} |
|||
@ -0,0 +1,391 @@ |
|||
package coscrypto_test |
|||
|
|||
import ( |
|||
"bytes" |
|||
"context" |
|||
"crypto/aes" |
|||
"crypto/cipher" |
|||
"crypto/md5" |
|||
"crypto/rand" |
|||
"encoding/base64" |
|||
"fmt" |
|||
"github.com/stretchr/testify/assert" |
|||
"github.com/tencentyun/cos-go-sdk-v5" |
|||
"github.com/tencentyun/cos-go-sdk-v5/crypto" |
|||
"io" |
|||
"io/ioutil" |
|||
math_rand "math/rand" |
|||
"net/http" |
|||
"net/url" |
|||
"os" |
|||
"sort" |
|||
"sync" |
|||
"time" |
|||
) |
|||
|
|||
func (s *CosTestSuite) TestMultiUpload_Normal() { |
|||
name := "test/ObjectPut" + time.Now().Format(time.RFC3339) |
|||
contentLength := int64(1024*1024*10 + 1) |
|||
originData := make([]byte, contentLength) |
|||
_, err := rand.Read(originData) |
|||
|
|||
cryptoCtx := coscrypto.CryptoContext{ |
|||
DataSize: contentLength, |
|||
PartSize: (contentLength / 16 / 3) * 16, |
|||
} |
|||
v, _, err := s.CClient.Object.InitiateMultipartUpload(context.Background(), name, nil, &cryptoCtx) |
|||
assert.Nil(s.T(), err, "Init Failed") |
|||
chunks, _, err := cos.SplitSizeIntoChunks(contentLength, cryptoCtx.PartSize) |
|||
assert.Nil(s.T(), err, "Split Failed") |
|||
optcom := &cos.CompleteMultipartUploadOptions{} |
|||
for _, chunk := range chunks { |
|||
opt := &cos.ObjectUploadPartOptions{ |
|||
ContentLength: chunk.Size, |
|||
} |
|||
f := bytes.NewReader(originData[chunk.OffSet : chunk.OffSet+chunk.Size]) |
|||
resp, err := s.CClient.Object.UploadPart(context.Background(), name, v.UploadID, chunk.Number, io.LimitReader(f, chunk.Size), opt, &cryptoCtx) |
|||
assert.Nil(s.T(), err, "UploadPart failed") |
|||
optcom.Parts = append(optcom.Parts, cos.Object{ |
|||
PartNumber: chunk.Number, ETag: resp.Header.Get("ETag"), |
|||
}) |
|||
} |
|||
_, _, err = s.CClient.Object.CompleteMultipartUpload(context.Background(), name, v.UploadID, optcom) |
|||
assert.Nil(s.T(), err, "Complete Failed") |
|||
|
|||
resp, err := s.CClient.Object.Get(context.Background(), name, nil) |
|||
assert.Nil(s.T(), err, "GetObject Failed") |
|||
defer resp.Body.Close() |
|||
decryptedData, _ := ioutil.ReadAll(resp.Body) |
|||
assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData") |
|||
|
|||
_, err = s.CClient.Object.Delete(context.Background(), name) |
|||
assert.Nil(s.T(), err, "DeleteObject Failed") |
|||
} |
|||
|
|||
func (s *CosTestSuite) TestMultiUpload_DecryptWithKey() { |
|||
name := "test/ObjectPut" + time.Now().Format(time.RFC3339) |
|||
contentLength := int64(1024*1024*10 + 1) |
|||
originData := make([]byte, contentLength) |
|||
_, err := rand.Read(originData) |
|||
f := bytes.NewReader(originData) |
|||
|
|||
// 分块上传
|
|||
cryptoCtx := coscrypto.CryptoContext{ |
|||
DataSize: contentLength, |
|||
PartSize: (contentLength / 16 / 3) * 16, |
|||
} |
|||
v, _, err := s.CClient.Object.InitiateMultipartUpload(context.Background(), name, nil, &cryptoCtx) |
|||
assert.Nil(s.T(), err, "Init Failed") |
|||
chunks, _, err := cos.SplitSizeIntoChunks(contentLength, cryptoCtx.PartSize) |
|||
assert.Nil(s.T(), err, "Split Failed") |
|||
optcom := &cos.CompleteMultipartUploadOptions{} |
|||
for _, chunk := range chunks { |
|||
opt := &cos.ObjectUploadPartOptions{ |
|||
ContentLength: chunk.Size, |
|||
Listener: &cos.DefaultProgressListener{}, |
|||
} |
|||
resp, err := s.CClient.Object.UploadPart(context.Background(), name, v.UploadID, chunk.Number, io.LimitReader(f, chunk.Size), opt, &cryptoCtx) |
|||
assert.Nil(s.T(), err, "UploadPart failed") |
|||
optcom.Parts = append(optcom.Parts, cos.Object{ |
|||
PartNumber: chunk.Number, ETag: resp.Header.Get("ETag"), |
|||
}) |
|||
} |
|||
_, _, err = s.CClient.Object.CompleteMultipartUpload(context.Background(), name, v.UploadID, optcom) |
|||
assert.Nil(s.T(), err, "Complete Failed") |
|||
|
|||
// 正常读取
|
|||
resp, err := s.Client.Object.Get(context.Background(), name, nil) |
|||
assert.Nil(s.T(), err, "GetObject Failed") |
|||
defer resp.Body.Close() |
|||
encryptedData, _ := ioutil.ReadAll(resp.Body) |
|||
assert.NotEqual(s.T(), bytes.Compare(encryptedData, originData), 0, "encryptedData == originData") |
|||
|
|||
// 获取解密信息
|
|||
resp, err = s.CClient.Object.Head(context.Background(), name, nil) |
|||
assert.Nil(s.T(), err, "HeadObject Failed") |
|||
cipherKey := resp.Header.Get(coscrypto.COSClientSideEncryptionKey) |
|||
cipherKeybs, err := base64.StdEncoding.DecodeString(cipherKey) |
|||
assert.Nil(s.T(), err, "base64 Decode Failed") |
|||
cipherIV := resp.Header.Get(coscrypto.COSClientSideEncryptionStart) |
|||
cipherIVbs, err := base64.StdEncoding.DecodeString(cipherIV) |
|||
assert.Nil(s.T(), err, "base64 Decode Failed") |
|||
key, err := s.Master.Decrypt(cipherKeybs) |
|||
assert.Nil(s.T(), err, "Master Decrypt Failed") |
|||
iv, err := s.Master.Decrypt(cipherIVbs) |
|||
assert.Nil(s.T(), err, "Master Decrypt Failed") |
|||
|
|||
// 手动解密
|
|||
block, err := aes.NewCipher(key) |
|||
assert.Nil(s.T(), err, "NewCipher Failed") |
|||
decrypter := cipher.NewCTR(block, iv) |
|||
decryptedData := make([]byte, len(originData)) |
|||
decrypter.XORKeyStream(decryptedData, encryptedData) |
|||
assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData") |
|||
|
|||
_, err = s.CClient.Object.Delete(context.Background(), name) |
|||
assert.Nil(s.T(), err, "DeleteObject Failed") |
|||
} |
|||
|
|||
func (s *CosTestSuite) TestMultiUpload_PutFromFile() { |
|||
name := "test/ObjectPut" + time.Now().Format(time.RFC3339) |
|||
filepath := "tmpfile" + time.Now().Format(time.RFC3339) |
|||
newfile, err := os.Create(filepath) |
|||
assert.Nil(s.T(), err, "Create File Failed") |
|||
defer os.Remove(filepath) |
|||
|
|||
contentLength := int64(1024*1024*10 + 1) |
|||
originData := make([]byte, contentLength) |
|||
_, err = rand.Read(originData) |
|||
newfile.Write(originData) |
|||
newfile.Close() |
|||
|
|||
m := md5.New() |
|||
m.Write(originData) |
|||
contentMD5 := m.Sum(nil) |
|||
cryptoCtx := coscrypto.CryptoContext{ |
|||
DataSize: contentLength, |
|||
PartSize: (contentLength / 16 / 3) * 16, |
|||
} |
|||
v, _, err := s.CClient.Object.InitiateMultipartUpload(context.Background(), name, nil, &cryptoCtx) |
|||
assert.Nil(s.T(), err, "Init Failed") |
|||
_, chunks, _, err := cos.SplitFileIntoChunks(filepath, cryptoCtx.PartSize) |
|||
assert.Nil(s.T(), err, "Split Failed") |
|||
optcom := &cos.CompleteMultipartUploadOptions{} |
|||
var wg sync.WaitGroup |
|||
var mtx sync.Mutex |
|||
for _, chunk := range chunks { |
|||
wg.Add(1) |
|||
go func(chk cos.Chunk) { |
|||
defer wg.Done() |
|||
fd, err := os.Open(filepath) |
|||
assert.Nil(s.T(), err, "Open File Failed") |
|||
opt := &cos.ObjectUploadPartOptions{ |
|||
ContentLength: chk.Size, |
|||
} |
|||
fd.Seek(chk.OffSet, os.SEEK_SET) |
|||
resp, err := s.CClient.Object.UploadPart(context.Background(), name, v.UploadID, chk.Number, io.LimitReader(fd, chk.Size), opt, &cryptoCtx) |
|||
assert.Nil(s.T(), err, "UploadPart failed") |
|||
mtx.Lock() |
|||
optcom.Parts = append(optcom.Parts, cos.Object{ |
|||
PartNumber: chk.Number, ETag: resp.Header.Get("ETag"), |
|||
}) |
|||
mtx.Unlock() |
|||
}(chunk) |
|||
} |
|||
wg.Wait() |
|||
sort.Sort(cos.ObjectList(optcom.Parts)) |
|||
_, _, err = s.CClient.Object.CompleteMultipartUpload(context.Background(), name, v.UploadID, optcom) |
|||
assert.Nil(s.T(), err, "Complete Failed") |
|||
|
|||
downfile := "downfile" + time.Now().Format(time.RFC3339) |
|||
_, err = s.CClient.Object.GetToFile(context.Background(), name, downfile, nil) |
|||
assert.Nil(s.T(), err, "GetObject Failed") |
|||
|
|||
m = md5.New() |
|||
fd, err := os.Open(downfile) |
|||
assert.Nil(s.T(), err, "Open File Failed") |
|||
defer os.Remove(downfile) |
|||
defer fd.Close() |
|||
io.Copy(m, fd) |
|||
downContentMD5 := m.Sum(nil) |
|||
assert.Equal(s.T(), bytes.Compare(contentMD5, downContentMD5), 0, "decryptData != originData") |
|||
|
|||
_, err = s.CClient.Object.Delete(context.Background(), name) |
|||
assert.Nil(s.T(), err, "DeleteObject Failed") |
|||
} |
|||
|
|||
func (s *CosTestSuite) TestMultiUpload_GetWithRange() { |
|||
name := "test/ObjectPut" + time.Now().Format(time.RFC3339) |
|||
filepath := "tmpfile" + time.Now().Format(time.RFC3339) |
|||
newfile, err := os.Create(filepath) |
|||
assert.Nil(s.T(), err, "Create File Failed") |
|||
defer os.Remove(filepath) |
|||
|
|||
contentLength := int64(1024*1024*10 + 1) |
|||
originData := make([]byte, contentLength) |
|||
_, err = rand.Read(originData) |
|||
newfile.Write(originData) |
|||
newfile.Close() |
|||
|
|||
m := md5.New() |
|||
m.Write(originData) |
|||
contentMD5 := m.Sum(nil) |
|||
cryptoCtx := coscrypto.CryptoContext{ |
|||
DataSize: contentLength, |
|||
PartSize: (contentLength / 16 / 3) * 16, |
|||
} |
|||
iniopt := &cos.InitiateMultipartUploadOptions{ |
|||
&cos.ACLHeaderOptions{ |
|||
XCosACL: "private", |
|||
}, |
|||
&cos.ObjectPutHeaderOptions{ |
|||
ContentMD5: base64.StdEncoding.EncodeToString(contentMD5), |
|||
XCosMetaXXX: &http.Header{}, |
|||
}, |
|||
} |
|||
iniopt.XCosMetaXXX.Add("x-cos-meta-isEncrypted", "true") |
|||
|
|||
v, _, err := s.CClient.Object.InitiateMultipartUpload(context.Background(), name, iniopt, &cryptoCtx) |
|||
assert.Nil(s.T(), err, "Init Failed") |
|||
_, chunks, _, err := cos.SplitFileIntoChunks(filepath, cryptoCtx.PartSize) |
|||
assert.Nil(s.T(), err, "Split Failed") |
|||
optcom := &cos.CompleteMultipartUploadOptions{} |
|||
var wg sync.WaitGroup |
|||
var mtx sync.Mutex |
|||
for _, chunk := range chunks { |
|||
wg.Add(1) |
|||
go func(chk cos.Chunk) { |
|||
defer wg.Done() |
|||
fd, err := os.Open(filepath) |
|||
assert.Nil(s.T(), err, "Open File Failed") |
|||
opt := &cos.ObjectUploadPartOptions{ |
|||
ContentLength: chk.Size, |
|||
} |
|||
fd.Seek(chk.OffSet, os.SEEK_SET) |
|||
resp, err := s.CClient.Object.UploadPart(context.Background(), name, v.UploadID, chk.Number, io.LimitReader(fd, chk.Size), opt, &cryptoCtx) |
|||
assert.Nil(s.T(), err, "UploadPart failed") |
|||
mtx.Lock() |
|||
optcom.Parts = append(optcom.Parts, cos.Object{ |
|||
PartNumber: chk.Number, ETag: resp.Header.Get("ETag"), |
|||
}) |
|||
mtx.Unlock() |
|||
}(chunk) |
|||
} |
|||
wg.Wait() |
|||
sort.Sort(cos.ObjectList(optcom.Parts)) |
|||
_, _, err = s.CClient.Object.CompleteMultipartUpload(context.Background(), name, v.UploadID, optcom) |
|||
assert.Nil(s.T(), err, "Complete Failed") |
|||
|
|||
// Range解密读取
|
|||
for i := 0; i < 10; i++ { |
|||
math_rand.Seed(time.Now().UnixNano()) |
|||
rangeStart := math_rand.Int63n(contentLength) |
|||
rangeEnd := rangeStart + math_rand.Int63n(contentLength-rangeStart) |
|||
if rangeEnd == rangeStart || rangeStart >= contentLength-1 { |
|||
continue |
|||
} |
|||
opt := &cos.ObjectGetOptions{ |
|||
Range: fmt.Sprintf("bytes=%v-%v", rangeStart, rangeEnd), |
|||
Listener: &cos.DefaultProgressListener{}, |
|||
} |
|||
resp, err := s.CClient.Object.Get(context.Background(), name, opt) |
|||
assert.Nil(s.T(), err, "GetObject Failed") |
|||
defer resp.Body.Close() |
|||
decryptedData, _ := ioutil.ReadAll(resp.Body) |
|||
assert.Equal(s.T(), bytes.Compare(originData[rangeStart:rangeEnd+1], decryptedData), 0, "decryptData != originData") |
|||
} |
|||
|
|||
opt := &cos.ObjectGetOptions{ |
|||
Listener: &cos.DefaultProgressListener{}, |
|||
} |
|||
resp, err := s.CClient.Object.Get(context.Background(), name, opt) |
|||
assert.Nil(s.T(), err, "GetObject Failed") |
|||
assert.Equal(s.T(), resp.Header.Get("x-cos-meta-isEncrypted"), "true", "meta data isn't consistent") |
|||
assert.Equal(s.T(), resp.Header.Get(coscrypto.COSClientSideEncryptionCekAlg), coscrypto.AesCtrAlgorithm, "meta data isn't consistent") |
|||
assert.Equal(s.T(), resp.Header.Get(coscrypto.COSClientSideEncryptionWrapAlg), coscrypto.CosKmsCryptoWrap, "meta data isn't consistent") |
|||
assert.Equal(s.T(), resp.Header.Get(coscrypto.COSClientSideEncryptionUnencryptedContentMD5), base64.StdEncoding.EncodeToString(contentMD5), "meta data isn't consistent") |
|||
defer resp.Body.Close() |
|||
decryptedData, _ := ioutil.ReadAll(resp.Body) |
|||
|
|||
assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData") |
|||
|
|||
_, err = s.CClient.Object.Delete(context.Background(), name) |
|||
assert.Nil(s.T(), err, "DeleteObject Failed") |
|||
} |
|||
|
|||
func (s *CosTestSuite) TestMultiUpload_GetWithNewClient() { |
|||
name := "test/ObjectPut" + time.Now().Format(time.RFC3339) |
|||
contentLength := int64(1024*1024*10 + 1) |
|||
originData := make([]byte, contentLength) |
|||
_, err := rand.Read(originData) |
|||
|
|||
cryptoCtx := coscrypto.CryptoContext{ |
|||
DataSize: contentLength, |
|||
PartSize: (contentLength / 16 / 3) * 16, |
|||
} |
|||
v, _, err := s.CClient.Object.InitiateMultipartUpload(context.Background(), name, nil, &cryptoCtx) |
|||
assert.Nil(s.T(), err, "Init Failed") |
|||
chunks, _, err := cos.SplitSizeIntoChunks(contentLength, cryptoCtx.PartSize) |
|||
assert.Nil(s.T(), err, "Split Failed") |
|||
optcom := &cos.CompleteMultipartUploadOptions{} |
|||
for _, chunk := range chunks { |
|||
opt := &cos.ObjectUploadPartOptions{ |
|||
ContentLength: chunk.Size, |
|||
} |
|||
f := bytes.NewReader(originData[chunk.OffSet : chunk.OffSet+chunk.Size]) |
|||
resp, err := s.CClient.Object.UploadPart(context.Background(), name, v.UploadID, chunk.Number, io.LimitReader(f, chunk.Size), opt, &cryptoCtx) |
|||
assert.Nil(s.T(), err, "UploadPart failed") |
|||
optcom.Parts = append(optcom.Parts, cos.Object{ |
|||
PartNumber: chunk.Number, ETag: resp.Header.Get("ETag"), |
|||
}) |
|||
} |
|||
_, _, err = s.CClient.Object.CompleteMultipartUpload(context.Background(), name, v.UploadID, optcom) |
|||
assert.Nil(s.T(), err, "Complete Failed") |
|||
|
|||
u, _ := url.Parse("https://" + kBucket + ".cos." + kRegion + ".myqcloud.com") |
|||
b := &cos.BaseURL{BucketURL: u} |
|||
c := cos.NewClient(b, &http.Client{ |
|||
Transport: &cos.AuthorizationTransport{ |
|||
SecretID: os.Getenv("COS_SECRETID"), |
|||
SecretKey: os.Getenv("COS_SECRETKEY"), |
|||
}, |
|||
}) |
|||
{ |
|||
// 使用不同的MatDesc客户端读取, 期待错误
|
|||
material := make(map[string]string) |
|||
material["desc"] = "cos crypto suite test 2" |
|||
kmsclient, _ := coscrypto.NewKMSClient(c.GetCredential(), kRegion) |
|||
master, _ := coscrypto.CreateMasterKMS(kmsclient, os.Getenv("KMSID"), material) |
|||
client := coscrypto.NewCryptoClient(c, master) |
|||
resp, err := client.Object.Get(context.Background(), name, nil) |
|||
assert.Nil(s.T(), resp, "Get Object Failed") |
|||
assert.NotNil(s.T(), err, "Get Object Failed") |
|||
} |
|||
|
|||
{ |
|||
// 使用相同的MatDesc客户端读取, 但KMSID不一样，期待正确，kms解密是不需要KMSID
|
|||
material := make(map[string]string) |
|||
material["desc"] = "cos crypto suite test" |
|||
kmsclient, _ := coscrypto.NewKMSClient(c.GetCredential(), kRegion) |
|||
master, _ := coscrypto.CreateMasterKMS(kmsclient, "KMSID", material) |
|||
client := coscrypto.NewCryptoClient(c, master) |
|||
resp, err := client.Object.Get(context.Background(), name, nil) |
|||
assert.Nil(s.T(), err, "Get Object Failed") |
|||
defer resp.Body.Close() |
|||
decryptedData, _ := ioutil.ReadAll(resp.Body) |
|||
assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData") |
|||
} |
|||
|
|||
{ |
|||
// 使用相同的MatDesc客户端读取, 地域不一样，期待错误
|
|||
material := make(map[string]string) |
|||
material["desc"] = "cos crypto suite test" |
|||
diffRegion := "ap-shanghai" |
|||
if diffRegion == kRegion { |
|||
diffRegion = "ap-guangzhou" |
|||
} |
|||
kmsclient, _ := coscrypto.NewKMSClient(c.GetCredential(), diffRegion) |
|||
master, _ := coscrypto.CreateMasterKMS(kmsclient, "KMSID", material) |
|||
client := coscrypto.NewCryptoClient(c, master) |
|||
resp, err := client.Object.Get(context.Background(), name, nil) |
|||
assert.Nil(s.T(), resp, "Get Object Failed") |
|||
assert.NotNil(s.T(), err, "Get Object Failed") |
|||
} |
|||
|
|||
{ |
|||
// 使用相同的MatDesc和KMSID客户端读取, 期待正确
|
|||
material := make(map[string]string) |
|||
material["desc"] = "cos crypto suite test" |
|||
kmsclient, _ := coscrypto.NewKMSClient(c.GetCredential(), kRegion) |
|||
master, _ := coscrypto.CreateMasterKMS(kmsclient, os.Getenv("KMSID"), material) |
|||
client := coscrypto.NewCryptoClient(c, master) |
|||
resp, err := client.Object.Get(context.Background(), name, nil) |
|||
assert.Nil(s.T(), err, "Get Object Failed") |
|||
defer resp.Body.Close() |
|||
decryptedData, _ := ioutil.ReadAll(resp.Body) |
|||
assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData") |
|||
} |
|||
|
|||
_, err = s.CClient.Object.Delete(context.Background(), name) |
|||
assert.Nil(s.T(), err, "DeleteObject Failed") |
|||
} |
|||
@ -0,0 +1,544 @@ |
|||
package coscrypto_test |
|||
|
|||
import ( |
|||
"bytes" |
|||
"context" |
|||
"crypto/aes" |
|||
"crypto/cipher" |
|||
"crypto/md5" |
|||
"crypto/rand" |
|||
"encoding/base64" |
|||
"fmt" |
|||
"github.com/stretchr/testify/assert" |
|||
"github.com/stretchr/testify/suite" |
|||
"github.com/tencentyun/cos-go-sdk-v5" |
|||
"github.com/tencentyun/cos-go-sdk-v5/crypto" |
|||
"io" |
|||
"io/ioutil" |
|||
math_rand "math/rand" |
|||
"net/http" |
|||
"net/url" |
|||
"os" |
|||
"testing" |
|||
"time" |
|||
) |
|||
|
|||
const ( |
|||
kAppid = 1259654469 |
|||
kBucket = "cosgosdktest-1259654469" |
|||
kRegion = "ap-guangzhou" |
|||
) |
|||
|
|||
type CosTestSuite struct { |
|||
suite.Suite |
|||
Client *cos.Client |
|||
CClient *coscrypto.CryptoClient |
|||
Master coscrypto.MasterCipher |
|||
} |
|||
|
|||
func (s *CosTestSuite) SetupSuite() { |
|||
u, _ := url.Parse("https://" + kBucket + ".cos." + kRegion + ".myqcloud.com") |
|||
b := &cos.BaseURL{BucketURL: u} |
|||
s.Client = cos.NewClient(b, &http.Client{ |
|||
Transport: &cos.AuthorizationTransport{ |
|||
SecretID: os.Getenv("COS_SECRETID"), |
|||
SecretKey: os.Getenv("COS_SECRETKEY"), |
|||
}, |
|||
}) |
|||
material := make(map[string]string) |
|||
material["desc"] = "cos crypto suite test" |
|||
kmsclient, _ := coscrypto.NewKMSClient(s.Client.GetCredential(), kRegion) |
|||
s.Master, _ = coscrypto.CreateMasterKMS(kmsclient, os.Getenv("KMSID"), material) |
|||
s.CClient = coscrypto.NewCryptoClient(s.Client, s.Master) |
|||
opt := &cos.BucketPutOptions{ |
|||
XCosACL: "public-read", |
|||
} |
|||
r, err := s.Client.Bucket.Put(context.Background(), opt) |
|||
if err != nil && r != nil && r.StatusCode == 409 { |
|||
fmt.Println("BucketAlreadyOwnedByYou") |
|||
} else if err != nil { |
|||
assert.Nil(s.T(), err, "PutBucket Failed") |
|||
} |
|||
} |
|||
|
|||
func (s *CosTestSuite) TestPutGetDeleteObject_DecryptWithKey_10MB() { |
|||
name := "test/objectPut" + time.Now().Format(time.RFC3339) |
|||
originData := make([]byte, 1024*1024*10+1) |
|||
_, err := rand.Read(originData) |
|||
f := bytes.NewReader(originData) |
|||
|
|||
// 加密存储
|
|||
_, err = s.CClient.Object.Put(context.Background(), name, f, nil) |
|||
assert.Nil(s.T(), err, "PutObject Failed") |
|||
|
|||
// 获取解密信息
|
|||
resp, err := s.CClient.Object.Head(context.Background(), name, nil) |
|||
assert.Nil(s.T(), err, "HeadObject Failed") |
|||
cipherKey := resp.Header.Get(coscrypto.COSClientSideEncryptionKey) |
|||
cipherKeybs, err := base64.StdEncoding.DecodeString(cipherKey) |
|||
assert.Nil(s.T(), err, "base64 Decode Failed") |
|||
cipherIV := resp.Header.Get(coscrypto.COSClientSideEncryptionStart) |
|||
cipherIVbs, err := base64.StdEncoding.DecodeString(cipherIV) |
|||
assert.Nil(s.T(), err, "base64 Decode Failed") |
|||
key, err := s.Master.Decrypt(cipherKeybs) |
|||
assert.Nil(s.T(), err, "Master Decrypt Failed") |
|||
iv, err := s.Master.Decrypt(cipherIVbs) |
|||
assert.Nil(s.T(), err, "Master Decrypt Failed") |
|||
|
|||
// 正常读取
|
|||
resp, err = s.Client.Object.Get(context.Background(), name, nil) |
|||
assert.Nil(s.T(), err, "GetObject Failed") |
|||
defer resp.Body.Close() |
|||
encryptedData, _ := ioutil.ReadAll(resp.Body) |
|||
assert.NotEqual(s.T(), bytes.Compare(encryptedData, originData), 0, "encryptedData == originData") |
|||
|
|||
// 手动解密
|
|||
block, err := aes.NewCipher(key) |
|||
assert.Nil(s.T(), err, "NewCipher Failed") |
|||
decrypter := cipher.NewCTR(block, iv) |
|||
decryptedData := make([]byte, len(originData)) |
|||
decrypter.XORKeyStream(decryptedData, encryptedData) |
|||
assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData") |
|||
_, err = s.CClient.Object.Delete(context.Background(), name) |
|||
assert.Nil(s.T(), err, "DeleteObject Failed") |
|||
} |
|||
|
|||
func (s *CosTestSuite) TestPutGetDeleteObject_Normal_10MB() { |
|||
name := "test/objectPut" + time.Now().Format(time.RFC3339) |
|||
originData := make([]byte, 1024*1024*10+1) |
|||
_, err := rand.Read(originData) |
|||
f := bytes.NewReader(originData) |
|||
|
|||
// 加密存储
|
|||
_, err = s.CClient.Object.Put(context.Background(), name, f, nil) |
|||
assert.Nil(s.T(), err, "PutObject Failed") |
|||
|
|||
// 解密读取
|
|||
resp, err := s.CClient.Object.Get(context.Background(), name, nil) |
|||
assert.Nil(s.T(), err, "GetObject Failed") |
|||
defer resp.Body.Close() |
|||
decryptedData, _ := ioutil.ReadAll(resp.Body) |
|||
assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData") |
|||
|
|||
_, err = s.CClient.Object.Delete(context.Background(), name) |
|||
assert.Nil(s.T(), err, "DeleteObject Failed") |
|||
} |
|||
|
|||
func (s *CosTestSuite) TestPutGetDeleteObject_VersionID() { |
|||
name := "test/objectPut" + time.Now().Format(time.RFC3339) |
|||
originData := make([]byte, 1024*1024*10+1) |
|||
_, err := rand.Read(originData) |
|||
f := bytes.NewReader(originData) |
|||
|
|||
opt := &cos.BucketPutVersionOptions{ |
|||
Status: "Enabled", |
|||
} |
|||
_, err = s.CClient.Bucket.PutVersioning(context.Background(), opt) |
|||
assert.Nil(s.T(), err, "PutVersioning Failed") |
|||
time.Sleep(3 * time.Second) |
|||
|
|||
// 加密存储
|
|||
resp, err := s.CClient.Object.Put(context.Background(), name, f, nil) |
|||
assert.Nil(s.T(), err, "PutObject Failed") |
|||
versionId := resp.Header.Get("x-cos-version-id") |
|||
|
|||
_, err = s.CClient.Object.Delete(context.Background(), name) |
|||
assert.Nil(s.T(), err, "DeleteObject Failed") |
|||
|
|||
// 解密读取
|
|||
resp, err = s.CClient.Object.Get(context.Background(), name, nil, versionId) |
|||
assert.Nil(s.T(), err, "GetObject Failed") |
|||
defer resp.Body.Close() |
|||
decryptedData, _ := ioutil.ReadAll(resp.Body) |
|||
assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData") |
|||
|
|||
delopt := &cos.ObjectDeleteOptions{ |
|||
VersionId: versionId, |
|||
} |
|||
_, err = s.CClient.Object.Delete(context.Background(), name, delopt) |
|||
assert.Nil(s.T(), err, "DeleteObject Failed") |
|||
|
|||
opt.Status = "Suspended" |
|||
_, err = s.CClient.Bucket.PutVersioning(context.Background(), opt) |
|||
assert.Nil(s.T(), err, "PutVersioning Failed") |
|||
} |
|||
|
|||
func (s *CosTestSuite) TestPutGetDeleteObject_ZeroFile() { |
|||
name := "test/objectPut" + time.Now().Format(time.RFC3339) |
|||
// 加密存储
|
|||
_, err := s.CClient.Object.Put(context.Background(), name, bytes.NewReader([]byte("")), nil) |
|||
assert.Nil(s.T(), err, "PutObject Failed") |
|||
|
|||
// 解密读取
|
|||
resp, err := s.CClient.Object.Get(context.Background(), name, nil) |
|||
assert.Nil(s.T(), err, "GetObject Failed") |
|||
defer resp.Body.Close() |
|||
decryptedData, _ := ioutil.ReadAll(resp.Body) |
|||
assert.Equal(s.T(), bytes.Compare([]byte(""), decryptedData), 0, "decryptData != originData") |
|||
|
|||
_, err = s.CClient.Object.Delete(context.Background(), name) |
|||
assert.Nil(s.T(), err, "DeleteObject Failed") |
|||
} |
|||
|
|||
func (s *CosTestSuite) TestPutGetDeleteObject_WithMetaData() { |
|||
name := "test/objectPut" + time.Now().Format(time.RFC3339) |
|||
originData := make([]byte, 1024*1024*10+1) |
|||
_, err := rand.Read(originData) |
|||
f := bytes.NewReader(originData) |
|||
|
|||
m := md5.New() |
|||
m.Write(originData) |
|||
contentMD5 := m.Sum(nil) |
|||
opt := &cos.ObjectPutOptions{ |
|||
&cos.ACLHeaderOptions{ |
|||
XCosACL: "private", |
|||
}, |
|||
&cos.ObjectPutHeaderOptions{ |
|||
ContentLength: 1024*1024*10 + 1, |
|||
ContentMD5: base64.StdEncoding.EncodeToString(contentMD5), |
|||
XCosMetaXXX: &http.Header{}, |
|||
}, |
|||
} |
|||
opt.XCosMetaXXX.Add("x-cos-meta-isEncrypted", "true") |
|||
// 加密存储
|
|||
_, err = s.CClient.Object.Put(context.Background(), name, f, opt) |
|||
assert.Nil(s.T(), err, "PutObject Failed") |
|||
|
|||
// 解密读取
|
|||
resp, err := s.CClient.Object.Get(context.Background(), name, nil) |
|||
assert.Nil(s.T(), err, "GetObject Failed") |
|||
defer resp.Body.Close() |
|||
decryptedData, _ := ioutil.ReadAll(resp.Body) |
|||
assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData") |
|||
assert.Equal(s.T(), resp.Header.Get("x-cos-meta-isEncrypted"), "true", "meta data isn't consistent") |
|||
assert.Equal(s.T(), resp.Header.Get(coscrypto.COSClientSideEncryptionCekAlg), coscrypto.AesCtrAlgorithm, "meta data isn't consistent") |
|||
assert.Equal(s.T(), resp.Header.Get(coscrypto.COSClientSideEncryptionWrapAlg), coscrypto.CosKmsCryptoWrap, "meta data isn't consistent") |
|||
assert.Equal(s.T(), resp.Header.Get(coscrypto.COSClientSideEncryptionUnencryptedContentMD5), base64.StdEncoding.EncodeToString(contentMD5), "meta data isn't consistent") |
|||
_, err = s.CClient.Object.Delete(context.Background(), name) |
|||
assert.Nil(s.T(), err, "DeleteObject Failed") |
|||
} |
|||
|
|||
func (s *CosTestSuite) TestPutGetDeleteObject_ByFile() { |
|||
name := "test/objectPut" + time.Now().Format(time.RFC3339) |
|||
filepath := "tmpfile" + time.Now().Format(time.RFC3339) |
|||
newfile, err := os.Create(filepath) |
|||
assert.Nil(s.T(), err, "Create File Failed") |
|||
defer os.Remove(filepath) |
|||
|
|||
originData := make([]byte, 1024*1024*10+1) |
|||
_, err = rand.Read(originData) |
|||
newfile.Write(originData) |
|||
newfile.Close() |
|||
|
|||
m := md5.New() |
|||
m.Write(originData) |
|||
contentMD5 := m.Sum(nil) |
|||
opt := &cos.ObjectPutOptions{ |
|||
&cos.ACLHeaderOptions{ |
|||
XCosACL: "private", |
|||
}, |
|||
&cos.ObjectPutHeaderOptions{ |
|||
ContentLength: 1024*1024*10 + 1, |
|||
ContentMD5: base64.StdEncoding.EncodeToString(contentMD5), |
|||
XCosMetaXXX: &http.Header{}, |
|||
}, |
|||
} |
|||
opt.XCosMetaXXX.Add("x-cos-meta-isEncrypted", "true") |
|||
// 加密存储
|
|||
_, err = s.CClient.Object.PutFromFile(context.Background(), name, filepath, opt) |
|||
assert.Nil(s.T(), err, "PutFromFile Failed") |
|||
|
|||
// 解密读取
|
|||
downfile := "downfile" + time.Now().Format(time.RFC3339) |
|||
resp, err := s.CClient.Object.GetToFile(context.Background(), name, downfile, nil) |
|||
assert.Nil(s.T(), err, "GetToFile Failed") |
|||
assert.Equal(s.T(), resp.Header.Get("x-cos-meta-isEncrypted"), "true", "meta data isn't consistent") |
|||
assert.Equal(s.T(), resp.Header.Get(coscrypto.COSClientSideEncryptionCekAlg), coscrypto.AesCtrAlgorithm, "meta data isn't consistent") |
|||
assert.Equal(s.T(), resp.Header.Get(coscrypto.COSClientSideEncryptionWrapAlg), coscrypto.CosKmsCryptoWrap, "meta data isn't consistent") |
|||
assert.Equal(s.T(), resp.Header.Get(coscrypto.COSClientSideEncryptionUnencryptedContentMD5), base64.StdEncoding.EncodeToString(contentMD5), "meta data isn't consistent") |
|||
|
|||
fd, err := os.Open(downfile) |
|||
assert.Nil(s.T(), err, "Open File Failed") |
|||
defer os.Remove(downfile) |
|||
defer fd.Close() |
|||
m = md5.New() |
|||
io.Copy(m, fd) |
|||
downContentMD5 := m.Sum(nil) |
|||
assert.Equal(s.T(), bytes.Compare(contentMD5, downContentMD5), 0, "decryptData != originData") |
|||
_, err = s.CClient.Object.Delete(context.Background(), name) |
|||
assert.Nil(s.T(), err, "DeleteObject Failed") |
|||
} |
|||
|
|||
func (s *CosTestSuite) TestPutGetDeleteObject_DecryptWithNewClient_10MB() { |
|||
name := "test/objectPut" + time.Now().Format(time.RFC3339) |
|||
originData := make([]byte, 1024*1024*10+1) |
|||
_, err := rand.Read(originData) |
|||
f := bytes.NewReader(originData) |
|||
|
|||
// 加密存储
|
|||
_, err = s.CClient.Object.Put(context.Background(), name, f, nil) |
|||
assert.Nil(s.T(), err, "PutObject Failed") |
|||
|
|||
u, _ := url.Parse("https://" + kBucket + ".cos." + kRegion + ".myqcloud.com") |
|||
b := &cos.BaseURL{BucketURL: u} |
|||
c := cos.NewClient(b, &http.Client{ |
|||
Transport: &cos.AuthorizationTransport{ |
|||
SecretID: os.Getenv("COS_SECRETID"), |
|||
SecretKey: os.Getenv("COS_SECRETKEY"), |
|||
}, |
|||
}) |
|||
{ |
|||
// 使用不同的MatDesc客户端读取, 期待错误
|
|||
material := make(map[string]string) |
|||
material["desc"] = "cos crypto suite test 2" |
|||
kmsclient, _ := coscrypto.NewKMSClient(c.GetCredential(), kRegion) |
|||
master, _ := coscrypto.CreateMasterKMS(kmsclient, os.Getenv("KMSID"), material) |
|||
client := coscrypto.NewCryptoClient(c, master) |
|||
resp, err := client.Object.Get(context.Background(), name, nil) |
|||
assert.Nil(s.T(), resp, "Get Object Failed") |
|||
assert.NotNil(s.T(), err, "Get Object Failed") |
|||
} |
|||
|
|||
{ |
|||
// 使用相同的MatDesc客户端读取, 但KMSID不一样，期待正确，kms解密是不需要KMSID
|
|||
material := make(map[string]string) |
|||
material["desc"] = "cos crypto suite test" |
|||
kmsclient, _ := coscrypto.NewKMSClient(s.Client.GetCredential(), kRegion) |
|||
master, _ := coscrypto.CreateMasterKMS(kmsclient, "KMSID", material) |
|||
client := coscrypto.NewCryptoClient(c, master) |
|||
resp, err := client.Object.Get(context.Background(), name, nil) |
|||
assert.Nil(s.T(), err, "Get Object Failed") |
|||
defer resp.Body.Close() |
|||
decryptedData, _ := ioutil.ReadAll(resp.Body) |
|||
assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData") |
|||
} |
|||
|
|||
{ |
|||
// 使用相同的MatDesc和KMSID客户端读取, 期待正确
|
|||
material := make(map[string]string) |
|||
material["desc"] = "cos crypto suite test" |
|||
kmsclient, _ := coscrypto.NewKMSClient(s.Client.GetCredential(), kRegion) |
|||
master, _ := coscrypto.CreateMasterKMS(kmsclient, os.Getenv("KMSID"), material) |
|||
client := coscrypto.NewCryptoClient(c, master) |
|||
resp, err := client.Object.Get(context.Background(), name, nil) |
|||
assert.Nil(s.T(), err, "Get Object Failed") |
|||
defer resp.Body.Close() |
|||
decryptedData, _ := ioutil.ReadAll(resp.Body) |
|||
assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData") |
|||
} |
|||
|
|||
_, err = s.CClient.Object.Delete(context.Background(), name) |
|||
assert.Nil(s.T(), err, "DeleteObject Failed") |
|||
} |
|||
|
|||
func (s *CosTestSuite) TestPutGetDeleteObject_RangeGet() { |
|||
name := "test/objectPut" + time.Now().Format(time.RFC3339) |
|||
contentLength := 1024*1024*10 + 1 |
|||
originData := make([]byte, contentLength) |
|||
_, err := rand.Read(originData) |
|||
f := bytes.NewReader(originData) |
|||
|
|||
// 加密存储
|
|||
_, err = s.CClient.Object.Put(context.Background(), name, f, nil) |
|||
assert.Nil(s.T(), err, "PutObject Failed") |
|||
|
|||
// Range解密读取
|
|||
for i := 0; i < 10; i++ { |
|||
math_rand.Seed(time.Now().UnixNano()) |
|||
rangeStart := math_rand.Intn(contentLength) |
|||
rangeEnd := rangeStart + math_rand.Intn(contentLength-rangeStart) |
|||
if rangeEnd == rangeStart || rangeStart >= contentLength-1 { |
|||
continue |
|||
} |
|||
opt := &cos.ObjectGetOptions{ |
|||
Range: fmt.Sprintf("bytes=%v-%v", rangeStart, rangeEnd), |
|||
} |
|||
resp, err := s.CClient.Object.Get(context.Background(), name, opt) |
|||
assert.Nil(s.T(), err, "GetObject Failed") |
|||
defer resp.Body.Close() |
|||
decryptedData, _ := ioutil.ReadAll(resp.Body) |
|||
assert.Equal(s.T(), bytes.Compare(originData[rangeStart:rangeEnd+1], decryptedData), 0, "decryptData != originData") |
|||
} |
|||
|
|||
// 解密读取
|
|||
resp, err := s.CClient.Object.Get(context.Background(), name, nil) |
|||
assert.Nil(s.T(), err, "GetObject Failed") |
|||
defer resp.Body.Close() |
|||
decryptedData, _ := ioutil.ReadAll(resp.Body) |
|||
assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData") |
|||
|
|||
_, err = s.CClient.Object.Delete(context.Background(), name) |
|||
assert.Nil(s.T(), err, "DeleteObject Failed") |
|||
} |
|||
|
|||
func (s *CosTestSuite) TestPutGetDeleteObject_WithListenerAndRange() { |
|||
name := "test/objectPut" + time.Now().Format(time.RFC3339) |
|||
contentLength := 1024*1024*10 + 1 |
|||
originData := make([]byte, contentLength) |
|||
_, err := rand.Read(originData) |
|||
f := bytes.NewReader(originData) |
|||
|
|||
// 加密存储
|
|||
popt := &cos.ObjectPutOptions{ |
|||
nil, |
|||
&cos.ObjectPutHeaderOptions{ |
|||
Listener: &cos.DefaultProgressListener{}, |
|||
}, |
|||
} |
|||
_, err = s.CClient.Object.Put(context.Background(), name, f, popt) |
|||
assert.Nil(s.T(), err, "PutObject Failed") |
|||
|
|||
// Range解密读取
|
|||
for i := 0; i < 10; i++ { |
|||
math_rand.Seed(time.Now().UnixNano()) |
|||
rangeStart := math_rand.Intn(contentLength) |
|||
rangeEnd := rangeStart + math_rand.Intn(contentLength-rangeStart) |
|||
if rangeEnd == rangeStart || rangeStart >= contentLength-1 { |
|||
continue |
|||
} |
|||
opt := &cos.ObjectGetOptions{ |
|||
Range: fmt.Sprintf("bytes=%v-%v", rangeStart, rangeEnd), |
|||
Listener: &cos.DefaultProgressListener{}, |
|||
} |
|||
resp, err := s.CClient.Object.Get(context.Background(), name, opt) |
|||
assert.Nil(s.T(), err, "GetObject Failed") |
|||
defer resp.Body.Close() |
|||
decryptedData, _ := ioutil.ReadAll(resp.Body) |
|||
assert.Equal(s.T(), bytes.Compare(originData[rangeStart:rangeEnd+1], decryptedData), 0, "decryptData != originData") |
|||
} |
|||
// 解密读取
|
|||
opt := &cos.ObjectGetOptions{ |
|||
Listener: &cos.DefaultProgressListener{}, |
|||
} |
|||
resp, err := s.CClient.Object.Get(context.Background(), name, opt) |
|||
assert.Nil(s.T(), err, "GetObject Failed") |
|||
defer resp.Body.Close() |
|||
decryptedData, _ := ioutil.ReadAll(resp.Body) |
|||
assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData") |
|||
|
|||
_, err = s.CClient.Object.Delete(context.Background(), name) |
|||
assert.Nil(s.T(), err, "DeleteObject Failed") |
|||
} |
|||
|
|||
func (s *CosTestSuite) TestPutGetDeleteObject_Copy() { |
|||
name := "test/objectPut" + time.Now().Format(time.RFC3339) |
|||
contentLength := 1024*1024*10 + 1 |
|||
originData := make([]byte, contentLength) |
|||
_, err := rand.Read(originData) |
|||
f := bytes.NewReader(originData) |
|||
|
|||
// 加密存储
|
|||
popt := &cos.ObjectPutOptions{ |
|||
nil, |
|||
&cos.ObjectPutHeaderOptions{ |
|||
Listener: &cos.DefaultProgressListener{}, |
|||
}, |
|||
} |
|||
resp, err := s.CClient.Object.Put(context.Background(), name, f, popt) |
|||
assert.Nil(s.T(), err, "PutObject Failed") |
|||
encryptedDataCRC := resp.Header.Get("x-cos-hash-crc64ecma") |
|||
time.Sleep(3 * time.Second) |
|||
sourceURL := fmt.Sprintf("%s/%s", s.CClient.BaseURL.BucketURL.Host, name) |
|||
{ |
|||
// x-cos-metadata-directive必须为Copy，否则丢失加密信息，无法解密
|
|||
dest := "test/ObjectCopy1" + time.Now().Format(time.RFC3339) |
|||
res, _, err := s.CClient.Object.Copy(context.Background(), dest, sourceURL, nil) |
|||
assert.Nil(s.T(), err, "ObjectCopy Failed") |
|||
assert.Equal(s.T(), encryptedDataCRC, res.CRC64, "CRC isn't consistent, return:%v, want:%v", res.CRC64, encryptedDataCRC) |
|||
|
|||
// Range解密读取
|
|||
for i := 0; i < 3; i++ { |
|||
math_rand.Seed(time.Now().UnixNano()) |
|||
rangeStart := math_rand.Intn(contentLength) |
|||
rangeEnd := rangeStart + math_rand.Intn(contentLength-rangeStart) |
|||
if rangeEnd == rangeStart || rangeStart >= contentLength-1 { |
|||
continue |
|||
} |
|||
opt := &cos.ObjectGetOptions{ |
|||
Range: fmt.Sprintf("bytes=%v-%v", rangeStart, rangeEnd), |
|||
Listener: &cos.DefaultProgressListener{}, |
|||
} |
|||
resp, err := s.CClient.Object.Get(context.Background(), dest, opt) |
|||
assert.Nil(s.T(), err, "GetObject Failed") |
|||
defer resp.Body.Close() |
|||
decryptedData, _ := ioutil.ReadAll(resp.Body) |
|||
assert.Equal(s.T(), bytes.Compare(originData[rangeStart:rangeEnd+1], decryptedData), 0, "decryptData != originData") |
|||
} |
|||
// 解密读取
|
|||
resp, err := s.CClient.Object.Get(context.Background(), dest, nil) |
|||
assert.Nil(s.T(), err, "GetObject Failed") |
|||
defer resp.Body.Close() |
|||
decryptedData, _ := ioutil.ReadAll(resp.Body) |
|||
assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData") |
|||
|
|||
_, err = s.CClient.Object.Delete(context.Background(), dest) |
|||
assert.Nil(s.T(), err, "DeleteObject Failed") |
|||
} |
|||
{ |
|||
// x-cos-metadata-directive必须为Copy，否则丢失加密信息，无法解密
|
|||
opt := &cos.ObjectCopyOptions{ |
|||
&cos.ObjectCopyHeaderOptions{ |
|||
XCosMetadataDirective: "Replaced", |
|||
}, |
|||
nil, |
|||
} |
|||
dest := "test/ObjectCopy2" + time.Now().Format(time.RFC3339) |
|||
res, _, err := s.CClient.Object.Copy(context.Background(), dest, sourceURL, opt) |
|||
assert.Nil(s.T(), err, "ObjectCopy Failed") |
|||
assert.Equal(s.T(), encryptedDataCRC, res.CRC64, "CRC isn't consistent, return:%v, want:%v", res.CRC64, encryptedDataCRC) |
|||
|
|||
// 解密读取
|
|||
resp, err := s.CClient.Object.Get(context.Background(), dest, nil) |
|||
assert.Nil(s.T(), err, "GetObject Failed") |
|||
defer resp.Body.Close() |
|||
decryptedData, _ := ioutil.ReadAll(resp.Body) |
|||
assert.NotEqual(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData") |
|||
|
|||
_, err = s.CClient.Object.Delete(context.Background(), dest) |
|||
assert.Nil(s.T(), err, "DeleteObject Failed") |
|||
} |
|||
{ |
|||
// MultiCopy若是分块拷贝，则无法拷贝元数据
|
|||
dest := "test/ObjectCopy3" + time.Now().Format(time.RFC3339) |
|||
res, _, err := s.CClient.Object.MultiCopy(context.Background(), dest, sourceURL, nil) |
|||
assert.Nil(s.T(), err, "ObjectMultiCopy Failed") |
|||
assert.Equal(s.T(), encryptedDataCRC, res.CRC64, "CRC isn't consistent, return:%v, want:%v", res.CRC64, encryptedDataCRC) |
|||
// Range解密读取
|
|||
for i := 0; i < 3; i++ { |
|||
math_rand.Seed(time.Now().UnixNano()) |
|||
rangeStart := math_rand.Intn(contentLength) |
|||
rangeEnd := rangeStart + math_rand.Intn(contentLength-rangeStart) |
|||
if rangeEnd == rangeStart || rangeStart >= contentLength-1 { |
|||
continue |
|||
} |
|||
opt := &cos.ObjectGetOptions{ |
|||
Range: fmt.Sprintf("bytes=%v-%v", rangeStart, rangeEnd), |
|||
Listener: &cos.DefaultProgressListener{}, |
|||
} |
|||
resp, err := s.CClient.Object.Get(context.Background(), dest, opt) |
|||
assert.Nil(s.T(), err, "GetObject Failed") |
|||
defer resp.Body.Close() |
|||
decryptedData, _ := ioutil.ReadAll(resp.Body) |
|||
assert.Equal(s.T(), bytes.Compare(originData[rangeStart:rangeEnd+1], decryptedData), 0, "decryptData != originData") |
|||
} |
|||
// 解密读取
|
|||
resp, err := s.CClient.Object.Get(context.Background(), dest, nil) |
|||
assert.Nil(s.T(), err, "GetObject Failed") |
|||
defer resp.Body.Close() |
|||
decryptedData, _ := ioutil.ReadAll(resp.Body) |
|||
assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData") |
|||
|
|||
_, err = s.CClient.Object.Delete(context.Background(), dest) |
|||
assert.Nil(s.T(), err, "DeleteObject Failed") |
|||
} |
|||
|
|||
_, err = s.CClient.Object.Delete(context.Background(), name) |
|||
assert.Nil(s.T(), err, "DeleteObject Failed") |
|||
} |
|||
|
|||
func TestCosTestSuite(t *testing.T) { |
|||
suite.Run(t, new(CosTestSuite)) |
|||
} |
|||
|
|||
func (s *CosTestSuite) TearDownSuite() { |
|||
} |
|||
@ -0,0 +1,141 @@ |
|||
package coscrypto |
|||
|
|||
import ( |
|||
"crypto/rand" |
|||
"encoding/binary" |
|||
"fmt" |
|||
"io" |
|||
math_rand "math/rand" |
|||
"time" |
|||
) |
|||
|
|||
const ( |
|||
COSClientSideEncryptionKey string = "x-cos-meta-client-side-encryption-key" |
|||
COSClientSideEncryptionStart = "x-cos-meta-client-side-encryption-start" |
|||
COSClientSideEncryptionCekAlg = "x-cos-meta-client-side-encryption-cek-alg" |
|||
COSClientSideEncryptionWrapAlg = "x-cos-meta-client-side-encryption-wrap-alg" |
|||
COSClientSideEncryptionMatDesc = "x-cos-meta-client-side-encryption-matdesc" |
|||
COSClientSideEncryptionUnencryptedContentLength = "x-cos-meta-client-side-encryption-unencrypted-content-length" |
|||
COSClientSideEncryptionUnencryptedContentMD5 = "x-cos-meta-client-side-encryption-unencrypted-content-md5" |
|||
COSClientSideEncryptionDataSize = "x-cos-meta-client-side-encryption-data-size" |
|||
COSClientSideEncryptionPartSize = "x-cos-meta-client-side-encryption-part-size" |
|||
UserAgent = "User-Agent" |
|||
) |
|||
|
|||
const ( |
|||
CosKmsCryptoWrap = "KMS/TencentCloud" |
|||
AesCtrAlgorithm = "AES/CTR/NoPadding" |
|||
EncryptionUaSuffix = "COSEncryptionClient" |
|||
) |
|||
|
|||
type MasterCipher interface { |
|||
Encrypt([]byte) ([]byte, error) |
|||
Decrypt([]byte) ([]byte, error) |
|||
GetWrapAlgorithm() string |
|||
GetMatDesc() string |
|||
} |
|||
|
|||
type ContentCipherBuilder interface { |
|||
ContentCipher() (ContentCipher, error) |
|||
ContentCipherEnv(Envelope) (ContentCipher, error) |
|||
GetMatDesc() string |
|||
} |
|||
|
|||
type ContentCipher interface { |
|||
EncryptContent(io.Reader) (io.ReadCloser, error) |
|||
DecryptContent(io.Reader) (io.ReadCloser, error) |
|||
Clone(cd CipherData) (ContentCipher, error) |
|||
GetEncryptedLen(int64) int64 |
|||
GetCipherData() *CipherData |
|||
GetAlignLen() int |
|||
} |
|||
|
|||
type Envelope struct { |
|||
IV string |
|||
CipherKey string |
|||
MatDesc string |
|||
WrapAlg string |
|||
CEKAlg string |
|||
UnencryptedMD5 string |
|||
UnencryptedContentLen string |
|||
} |
|||
|
|||
func (el Envelope) IsValid() bool { |
|||
return len(el.IV) > 0 && |
|||
len(el.CipherKey) > 0 && |
|||
len(el.WrapAlg) > 0 && |
|||
len(el.CEKAlg) > 0 |
|||
} |
|||
|
|||
func (el Envelope) String() string { |
|||
return fmt.Sprintf("IV=%s&CipherKey=%s&WrapAlg=%s&CEKAlg=%s", el.IV, el.CipherKey, el.WrapAlg, el.CEKAlg) |
|||
} |
|||
|
|||
type CipherData struct { |
|||
IV []byte |
|||
Key []byte |
|||
MatDesc string |
|||
WrapAlgorithm string |
|||
CEKAlgorithm string |
|||
EncryptedIV []byte |
|||
EncryptedKey []byte |
|||
} |
|||
|
|||
func (cd *CipherData) RandomKeyIv(keyLen int, ivLen int) error { |
|||
math_rand.Seed(time.Now().UnixNano()) |
|||
|
|||
// Key
|
|||
cd.Key = make([]byte, keyLen) |
|||
if _, err := io.ReadFull(rand.Reader, cd.Key); err != nil { |
|||
return err |
|||
} |
|||
|
|||
// sizeof uint64
|
|||
if ivLen < 8 { |
|||
return fmt.Errorf("ivLen:%d less than 8", ivLen) |
|||
} |
|||
|
|||
// IV: | nonce: 8 bytes | Serial number: 8 bytes |
|
|||
cd.IV = make([]byte, ivLen) |
|||
if _, err := io.ReadFull(rand.Reader, cd.IV[0:ivLen-8]); err != nil { |
|||
return err |
|||
} |
|||
|
|||
// only use 4 byte,in order not to overflow when SeekIV()
|
|||
randNumber := math_rand.Uint32() |
|||
cd.SetIV(uint64(randNumber)) |
|||
return nil |
|||
} |
|||
|
|||
func (cd *CipherData) SetIV(iv uint64) { |
|||
ivLen := len(cd.IV) |
|||
binary.BigEndian.PutUint64(cd.IV[ivLen-8:], iv) |
|||
} |
|||
|
|||
func (cd *CipherData) GetIV() uint64 { |
|||
ivLen := len(cd.IV) |
|||
return binary.BigEndian.Uint64(cd.IV[ivLen-8:]) |
|||
} |
|||
|
|||
func (cd *CipherData) SeekIV(startPos uint64) { |
|||
cd.SetIV(cd.GetIV() + startPos/uint64(len(cd.IV))) |
|||
} |
|||
|
|||
func (cd *CipherData) Clone() CipherData { |
|||
var cloneCd CipherData |
|||
cloneCd = *cd |
|||
|
|||
cloneCd.Key = make([]byte, len(cd.Key)) |
|||
copy(cloneCd.Key, cd.Key) |
|||
|
|||
cloneCd.IV = make([]byte, len(cd.IV)) |
|||
copy(cloneCd.IV, cd.IV) |
|||
|
|||
cloneCd.EncryptedIV = make([]byte, len(cd.EncryptedIV)) |
|||
copy(cloneCd.EncryptedIV, cd.EncryptedIV) |
|||
|
|||
cloneCd.EncryptedKey = make([]byte, len(cd.EncryptedKey)) |
|||
copy(cloneCd.EncryptedKey, cd.EncryptedKey) |
|||
|
|||
return cloneCd |
|||
} |
|||
@ -0,0 +1,87 @@ |
|||
package coscrypto |
|||
|
|||
import ( |
|||
"encoding/base64" |
|||
"encoding/json" |
|||
"fmt" |
|||
|
|||
"github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common" |
|||
"github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/profile" |
|||
kms "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/kms/v20190118" |
|||
"github.com/tencentyun/cos-go-sdk-v5" |
|||
) |
|||
|
|||
const ( |
|||
KMSEndPoint = "kms.tencentcloudapi.com" |
|||
) |
|||
|
|||
type MasterKMSCipher struct { |
|||
Client *kms.Client |
|||
KmsId string |
|||
MatDesc string |
|||
} |
|||
|
|||
func NewKMSClient(cred *cos.Credential, region string) (*kms.Client, error) { |
|||
if cred == nil { |
|||
fmt.Errorf("credential is nil") |
|||
} |
|||
credential := common.NewTokenCredential( |
|||
cred.SecretID, |
|||
cred.SecretKey, |
|||
cred.SessionToken, |
|||
) |
|||
cpf := profile.NewClientProfile() |
|||
cpf.HttpProfile.Endpoint = KMSEndPoint |
|||
client, err := kms.NewClient(credential, region, cpf) |
|||
return client, err |
|||
} |
|||
|
|||
func CreateMasterKMS(client *kms.Client, kmsId string, desc map[string]string) (MasterCipher, error) { |
|||
if kmsId == "" || client == nil { |
|||
return nil, fmt.Errorf("KMS ID is empty or kms client is nil") |
|||
} |
|||
var kmsCipher MasterKMSCipher |
|||
var jdesc string |
|||
if len(desc) > 0 { |
|||
bs, err := json.Marshal(desc) |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
jdesc = string(bs) |
|||
} |
|||
kmsCipher.Client = client |
|||
kmsCipher.KmsId = kmsId |
|||
kmsCipher.MatDesc = jdesc |
|||
return &kmsCipher, nil |
|||
} |
|||
|
|||
func (kc *MasterKMSCipher) Encrypt(plaintext []byte) ([]byte, error) { |
|||
request := kms.NewEncryptRequest() |
|||
request.KeyId = common.StringPtr(kc.KmsId) |
|||
request.EncryptionContext = common.StringPtr(kc.MatDesc) |
|||
request.Plaintext = common.StringPtr(base64.StdEncoding.EncodeToString(plaintext)) |
|||
resp, err := kc.Client.Encrypt(request) |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
return []byte(*resp.Response.CiphertextBlob), nil |
|||
} |
|||
|
|||
func (kc *MasterKMSCipher) Decrypt(ciphertext []byte) ([]byte, error) { |
|||
request := kms.NewDecryptRequest() |
|||
request.CiphertextBlob = common.StringPtr(string(ciphertext)) |
|||
request.EncryptionContext = common.StringPtr(kc.MatDesc) |
|||
resp, err := kc.Client.Decrypt(request) |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
return base64.StdEncoding.DecodeString(*resp.Response.Plaintext) |
|||
} |
|||
|
|||
func (kc *MasterKMSCipher) GetWrapAlgorithm() string { |
|||
return CosKmsCryptoWrap |
|||
} |
|||
|
|||
func (kc *MasterKMSCipher) GetMatDesc() string { |
|||
return kc.MatDesc |
|||
} |
|||
@ -0,0 +1,91 @@ |
|||
package coscrypto_test |
|||
|
|||
import ( |
|||
"bytes" |
|||
"crypto/rand" |
|||
"encoding/base64" |
|||
"encoding/json" |
|||
"github.com/stretchr/testify/assert" |
|||
"github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common" |
|||
kms "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/kms/v20190118" |
|||
"github.com/tencentyun/cos-go-sdk-v5" |
|||
"github.com/tencentyun/cos-go-sdk-v5/crypto" |
|||
"os" |
|||
) |
|||
|
|||
func (s *CosTestSuite) TestMasterKmsCipher_TestKmsClient() { |
|||
kmsclient, _ := coscrypto.NewKMSClient(&cos.Credential{ |
|||
SecretID: os.Getenv("COS_SECRETID"), |
|||
SecretKey: os.Getenv("COS_SECRETKEY"), |
|||
}, kRegion) |
|||
|
|||
originData := make([]byte, 1024) |
|||
_, err := rand.Read(originData) |
|||
|
|||
ctx := make(map[string]string) |
|||
ctx["desc"] = string(originData[:10]) |
|||
bs, _ := json.Marshal(ctx) |
|||
ctxJson := string(bs) |
|||
enReq := kms.NewEncryptRequest() |
|||
enReq.KeyId = common.StringPtr(os.Getenv("KMSID")) |
|||
enReq.EncryptionContext = common.StringPtr(ctxJson) |
|||
enReq.Plaintext = common.StringPtr(base64.StdEncoding.EncodeToString(originData)) |
|||
enResp, err := kmsclient.Encrypt(enReq) |
|||
assert.Nil(s.T(), err, "Encrypt Failed") |
|||
encryptedData := []byte(*enResp.Response.CiphertextBlob) |
|||
|
|||
deReq := kms.NewDecryptRequest() |
|||
deReq.CiphertextBlob = common.StringPtr(string(encryptedData)) |
|||
deReq.EncryptionContext = common.StringPtr(ctxJson) |
|||
deResp, err := kmsclient.Decrypt(deReq) |
|||
assert.Nil(s.T(), err, "Decrypt Failed") |
|||
decryptedData, err := base64.StdEncoding.DecodeString(*deResp.Response.Plaintext) |
|||
assert.Nil(s.T(), err, "base64 Decode Failed") |
|||
assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "originData != decryptedData") |
|||
} |
|||
|
|||
func (s *CosTestSuite) TestMasterKmsCipher_TestNormal() { |
|||
kmsclient, _ := coscrypto.NewKMSClient(&cos.Credential{ |
|||
SecretID: os.Getenv("COS_SECRETID"), |
|||
SecretKey: os.Getenv("COS_SECRETKEY"), |
|||
}, kRegion) |
|||
|
|||
desc := make(map[string]string) |
|||
desc["test"] = "TestMasterKmsCipher_TestNormal" |
|||
master, err := coscrypto.CreateMasterKMS(kmsclient, os.Getenv("KMSID"), desc) |
|||
assert.Nil(s.T(), err, "CreateMasterKMS Failed") |
|||
|
|||
originData := make([]byte, 1024) |
|||
_, err = rand.Read(originData) |
|||
|
|||
encryptedData, err := master.Encrypt(originData) |
|||
assert.Nil(s.T(), err, "Encrypt Failed") |
|||
|
|||
decryptedData, err := master.Decrypt(encryptedData) |
|||
assert.Nil(s.T(), err, "Decrypt Failed") |
|||
|
|||
assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "originData != decryptedData") |
|||
} |
|||
|
|||
func (s *CosTestSuite) TestMasterKmsCipher_TestError() { |
|||
kmsclient, _ := coscrypto.NewKMSClient(&cos.Credential{ |
|||
SecretID: os.Getenv("COS_SECRETID"), |
|||
SecretKey: os.Getenv("COS_SECRETKEY"), |
|||
}, kRegion) |
|||
|
|||
desc := make(map[string]string) |
|||
desc["test"] = "TestMasterKmsCipher_TestNormal" |
|||
master, err := coscrypto.CreateMasterKMS(kmsclient, "ErrorKMSID", desc) |
|||
assert.Nil(s.T(), err, "CreateMasterKMS Failed") |
|||
|
|||
originData := make([]byte, 1024) |
|||
_, err = rand.Read(originData) |
|||
|
|||
encryptedData, err := master.Encrypt(originData) |
|||
assert.NotNil(s.T(), err, "Encrypt Failed") |
|||
|
|||
decryptedData, err := master.Decrypt(encryptedData) |
|||
assert.NotNil(s.T(), err, "Decrypt Failed") |
|||
|
|||
assert.NotEqual(s.T(), bytes.Compare(originData, decryptedData), 0, "originData != decryptedData") |
|||
} |
|||
@ -0,0 +1,316 @@ |
|||
package main |
|||
|
|||
import ( |
|||
"bytes" |
|||
"context" |
|||
"crypto/md5" |
|||
"crypto/rand" |
|||
"fmt" |
|||
"io" |
|||
"io/ioutil" |
|||
math_rand "math/rand" |
|||
"net/http" |
|||
"net/url" |
|||
"os" |
|||
"time" |
|||
|
|||
"github.com/tencentyun/cos-go-sdk-v5" |
|||
"github.com/tencentyun/cos-go-sdk-v5/crypto" |
|||
"github.com/tencentyun/cos-go-sdk-v5/debug" |
|||
) |
|||
|
|||
func log_status(err error) { |
|||
if err == nil { |
|||
return |
|||
} |
|||
if cos.IsNotFoundError(err) { |
|||
// WARN
|
|||
fmt.Println("WARN: Resource is not existed") |
|||
} else if e, ok := cos.IsCOSError(err); ok { |
|||
fmt.Printf("ERROR: Code: %v\n", e.Code) |
|||
fmt.Printf("ERROR: Message: %v\n", e.Message) |
|||
fmt.Printf("ERROR: Resource: %v\n", e.Resource) |
|||
fmt.Printf("ERROR: RequestId: %v\n", e.RequestID) |
|||
// ERROR
|
|||
} else { |
|||
fmt.Printf("ERROR: %v\n", err) |
|||
// ERROR
|
|||
} |
|||
os.Exit(1) |
|||
} |
|||
|
|||
func cos_max(x, y int64) int64 { |
|||
if x > y { |
|||
return x |
|||
} |
|||
return y |
|||
} |
|||
|
|||
func simple_put_object() { |
|||
u, _ := url.Parse("https://test-1259654469.cos.ap-guangzhou.myqcloud.com") |
|||
b := &cos.BaseURL{BucketURL: u} |
|||
c := cos.NewClient(b, &http.Client{ |
|||
Transport: &cos.AuthorizationTransport{ |
|||
SecretID: os.Getenv("COS_SECRETID"), |
|||
SecretKey: os.Getenv("COS_SECRETKEY"), |
|||
Transport: &debug.DebugRequestTransport{ |
|||
RequestHeader: true, |
|||
RequestBody: false, |
|||
ResponseHeader: true, |
|||
ResponseBody: false, |
|||
}, |
|||
}, |
|||
}) |
|||
// Case1 上传对象
|
|||
name := "test/example2" |
|||
|
|||
fmt.Println("============== simple_put_object ======================") |
|||
// 该标识信息唯一确认一个主加密密钥, 解密时，需要传入相同的标识信息
|
|||
// KMS加密时，该信息设置成EncryptionContext，最大支持1024字符，如果Encrypt指定了该参数，则在Decrypt 时需要提供同样的参数
|
|||
materialDesc := make(map[string]string) |
|||
//materialDesc["desc"] = "material information of your master encrypt key"
|
|||
|
|||
// 创建KMS客户端
|
|||
kmsclient, _ := coscrypto.NewKMSClient(c.GetCredential(), "ap-guangzhou") |
|||
// 创建KMS主加密密钥，标识信息和主密钥一一对应
|
|||
kmsID := os.Getenv("KMSID") |
|||
masterCipher, _ := coscrypto.CreateMasterKMS(kmsclient, kmsID, materialDesc) |
|||
// 创建加密客户端
|
|||
client := coscrypto.NewCryptoClient(c, masterCipher) |
|||
|
|||
contentLength := 1024*1024*10 + 1 |
|||
originData := make([]byte, contentLength) |
|||
_, err := rand.Read(originData) |
|||
f := bytes.NewReader(originData) |
|||
// 加密上传
|
|||
_, err = client.Object.Put(context.Background(), name, f, nil) |
|||
log_status(err) |
|||
|
|||
math_rand.Seed(time.Now().UnixNano()) |
|||
rangeStart := math_rand.Intn(contentLength) |
|||
rangeEnd := rangeStart + math_rand.Intn(contentLength-rangeStart) |
|||
opt := &cos.ObjectGetOptions{ |
|||
Range: fmt.Sprintf("bytes=%v-%v", rangeStart, rangeEnd), |
|||
} |
|||
// 解密下载
|
|||
resp, err := client.Object.Get(context.Background(), name, opt) |
|||
log_status(err) |
|||
defer resp.Body.Close() |
|||
decryptedData, _ := ioutil.ReadAll(resp.Body) |
|||
if bytes.Compare(decryptedData, originData[rangeStart:rangeEnd+1]) != 0 { |
|||
fmt.Println("Error: encryptedData != originData") |
|||
} |
|||
} |
|||
|
|||
func simple_put_object_from_file() { |
|||
u, _ := url.Parse("https://test-1259654469.cos.ap-guangzhou.myqcloud.com") |
|||
b := &cos.BaseURL{BucketURL: u} |
|||
c := cos.NewClient(b, &http.Client{ |
|||
Transport: &cos.AuthorizationTransport{ |
|||
SecretID: os.Getenv("COS_SECRETID"), |
|||
SecretKey: os.Getenv("COS_SECRETKEY"), |
|||
Transport: &debug.DebugRequestTransport{ |
|||
RequestHeader: true, |
|||
RequestBody: false, |
|||
ResponseHeader: true, |
|||
ResponseBody: false, |
|||
}, |
|||
}, |
|||
}) |
|||
// Case1 上传对象
|
|||
name := "test/example1" |
|||
|
|||
fmt.Println("============== simple_put_object_from_file ======================") |
|||
// 该标识信息唯一确认一个主加密密钥, 解密时，需要传入相同的标识信息
|
|||
// KMS加密时，该信息设置成EncryptionContext，最大支持1024字符，如果Encrypt指定了该参数，则在Decrypt 时需要提供同样的参数
|
|||
materialDesc := make(map[string]string) |
|||
//materialDesc["desc"] = "material information of your master encrypt key"
|
|||
|
|||
// 创建KMS客户端
|
|||
kmsclient, _ := coscrypto.NewKMSClient(c.GetCredential(), "ap-guangzhou") |
|||
// 创建KMS主加密密钥，标识信息和主密钥一一对应
|
|||
kmsID := os.Getenv("KMSID") |
|||
masterCipher, _ := coscrypto.CreateMasterKMS(kmsclient, kmsID, materialDesc) |
|||
// 创建加密客户端
|
|||
client := coscrypto.NewCryptoClient(c, masterCipher) |
|||
|
|||
filepath := "test" |
|||
fd, err := os.Open(filepath) |
|||
log_status(err) |
|||
defer fd.Close() |
|||
m := md5.New() |
|||
io.Copy(m, fd) |
|||
originDataMD5 := m.Sum(nil) |
|||
|
|||
// 加密上传
|
|||
_, err = client.Object.PutFromFile(context.Background(), name, filepath, nil) |
|||
log_status(err) |
|||
|
|||
// 解密下载
|
|||
_, err = client.Object.GetToFile(context.Background(), name, "./test.download", nil) |
|||
log_status(err) |
|||
|
|||
fd, err = os.Open("./test.download") |
|||
log_status(err) |
|||
defer fd.Close() |
|||
m = md5.New() |
|||
io.Copy(m, fd) |
|||
decryptedDataMD5 := m.Sum(nil) |
|||
|
|||
if bytes.Compare(decryptedDataMD5, originDataMD5) != 0 { |
|||
fmt.Println("Error: encryptedData != originData") |
|||
} |
|||
} |
|||
|
|||
func multi_put_object() { |
|||
u, _ := url.Parse("https://test-1259654469.cos.ap-guangzhou.myqcloud.com") |
|||
b := &cos.BaseURL{BucketURL: u} |
|||
c := cos.NewClient(b, &http.Client{ |
|||
Transport: &cos.AuthorizationTransport{ |
|||
SecretID: os.Getenv("COS_SECRETID"), |
|||
SecretKey: os.Getenv("COS_SECRETKEY"), |
|||
Transport: &debug.DebugRequestTransport{ |
|||
RequestHeader: true, |
|||
RequestBody: false, |
|||
ResponseHeader: true, |
|||
ResponseBody: false, |
|||
}, |
|||
}, |
|||
}) |
|||
// Case1 上传对象
|
|||
name := "test/example1" |
|||
|
|||
fmt.Println("============== multi_put_object ======================") |
|||
// 该标识信息唯一确认一个主加密密钥, 解密时，需要传入相同的标识信息
|
|||
// KMS加密时，该信息设置成EncryptionContext，最大支持1024字符，如果Encrypt指定了该参数，则在Decrypt 时需要提供同样的参数
|
|||
materialDesc := make(map[string]string) |
|||
//materialDesc["desc"] = "material information of your master encrypt key"
|
|||
|
|||
// 创建KMS客户端
|
|||
kmsclient, _ := coscrypto.NewKMSClient(c.GetCredential(), "ap-guangzhou") |
|||
// 创建KMS主加密密钥，标识信息和主密钥一一对应
|
|||
kmsID := os.Getenv("KMSID") |
|||
masterCipher, _ := coscrypto.CreateMasterKMS(kmsclient, kmsID, materialDesc) |
|||
// 创建加密客户端
|
|||
client := coscrypto.NewCryptoClient(c, masterCipher) |
|||
|
|||
contentLength := int64(1024*1024*10 + 1) |
|||
originData := make([]byte, contentLength) |
|||
_, err := rand.Read(originData) |
|||
log_status(err) |
|||
|
|||
// 分块上传
|
|||
cryptoCtx := coscrypto.CryptoContext{ |
|||
DataSize: contentLength, |
|||
// 每个分块需要16字节对齐
|
|||
PartSize: cos_max(1024*1024, (contentLength/16/3)*16), |
|||
} |
|||
v, _, err := client.Object.InitiateMultipartUpload(context.Background(), name, nil, &cryptoCtx) |
|||
log_status(err) |
|||
// 切分数据
|
|||
chunks, _, err := cos.SplitSizeIntoChunks(contentLength, cryptoCtx.PartSize) |
|||
log_status(err) |
|||
optcom := &cos.CompleteMultipartUploadOptions{} |
|||
for _, chunk := range chunks { |
|||
opt := &cos.ObjectUploadPartOptions{ |
|||
ContentLength: chunk.Size, |
|||
} |
|||
f := bytes.NewReader(originData[chunk.OffSet : chunk.OffSet+chunk.Size]) |
|||
resp, err := client.Object.UploadPart(context.Background(), name, v.UploadID, chunk.Number, f, opt, &cryptoCtx) |
|||
log_status(err) |
|||
optcom.Parts = append(optcom.Parts, cos.Object{ |
|||
PartNumber: chunk.Number, ETag: resp.Header.Get("ETag"), |
|||
}) |
|||
} |
|||
_, _, err = client.Object.CompleteMultipartUpload(context.Background(), name, v.UploadID, optcom) |
|||
log_status(err) |
|||
|
|||
resp, err := client.Object.Get(context.Background(), name, nil) |
|||
log_status(err) |
|||
defer resp.Body.Close() |
|||
decryptedData, _ := ioutil.ReadAll(resp.Body) |
|||
if bytes.Compare(decryptedData, originData) != 0 { |
|||
fmt.Println("Error: encryptedData != originData") |
|||
} |
|||
} |
|||
|
|||
func multi_put_object_from_file() { |
|||
u, _ := url.Parse("https://test-1259654469.cos.ap-guangzhou.myqcloud.com") |
|||
b := &cos.BaseURL{BucketURL: u} |
|||
c := cos.NewClient(b, &http.Client{ |
|||
Transport: &cos.AuthorizationTransport{ |
|||
SecretID: os.Getenv("COS_SECRETID"), |
|||
SecretKey: os.Getenv("COS_SECRETKEY"), |
|||
Transport: &debug.DebugRequestTransport{ |
|||
RequestHeader: true, |
|||
RequestBody: false, |
|||
ResponseHeader: true, |
|||
ResponseBody: false, |
|||
}, |
|||
}, |
|||
}) |
|||
// Case1 上传对象
|
|||
name := "test/example1" |
|||
|
|||
fmt.Println("============== multi_put_object_from_file ======================") |
|||
// 该标识信息唯一确认一个主加密密钥, 解密时，需要传入相同的标识信息
|
|||
// KMS加密时，该信息设置成EncryptionContext，最大支持1024字符，如果Encrypt指定了该参数，则在Decrypt 时需要提供同样的参数
|
|||
materialDesc := make(map[string]string) |
|||
//materialDesc["desc"] = "material information of your master encrypt key"
|
|||
|
|||
// 创建KMS客户端
|
|||
kmsclient, _ := coscrypto.NewKMSClient(c.GetCredential(), "ap-guangzhou") |
|||
// 创建KMS主加密密钥，标识信息和主密钥一一对应
|
|||
kmsID := os.Getenv("KMSID") |
|||
masterCipher, _ := coscrypto.CreateMasterKMS(kmsclient, kmsID, materialDesc) |
|||
// 创建加密客户端
|
|||
client := coscrypto.NewCryptoClient(c, masterCipher) |
|||
|
|||
filepath := "test" |
|||
stat, err := os.Stat(filepath) |
|||
log_status(err) |
|||
contentLength := stat.Size() |
|||
|
|||
// 分块上传
|
|||
cryptoCtx := coscrypto.CryptoContext{ |
|||
DataSize: contentLength, |
|||
// 每个分块需要16字节对齐
|
|||
PartSize: cos_max(1024*1024, (contentLength/16/3)*16), |
|||
} |
|||
// 切分数据
|
|||
_, chunks, _, err := cos.SplitFileIntoChunks(filepath, cryptoCtx.PartSize) |
|||
log_status(err) |
|||
|
|||
// init mulitupload
|
|||
v, _, err := client.Object.InitiateMultipartUpload(context.Background(), name, nil, &cryptoCtx) |
|||
log_status(err) |
|||
|
|||
// part upload
|
|||
optcom := &cos.CompleteMultipartUploadOptions{} |
|||
for _, chunk := range chunks { |
|||
fd, err := os.Open(filepath) |
|||
log_status(err) |
|||
opt := &cos.ObjectUploadPartOptions{ |
|||
ContentLength: chunk.Size, |
|||
} |
|||
fd.Seek(chunk.OffSet, os.SEEK_SET) |
|||
resp, err := client.Object.UploadPart(context.Background(), name, v.UploadID, chunk.Number, cos.LimitReadCloser(fd, chunk.Size), opt, &cryptoCtx) |
|||
log_status(err) |
|||
optcom.Parts = append(optcom.Parts, cos.Object{ |
|||
PartNumber: chunk.Number, ETag: resp.Header.Get("ETag"), |
|||
}) |
|||
} |
|||
// complete upload
|
|||
_, _, err = client.Object.CompleteMultipartUpload(context.Background(), name, v.UploadID, optcom) |
|||
log_status(err) |
|||
|
|||
_, err = client.Object.GetToFile(context.Background(), name, "test.download", nil) |
|||
log_status(err) |
|||
} |
|||
|
|||
func main() { |
|||
simple_put_object() |
|||
simple_put_object_from_file() |
|||
multi_put_object() |
|||
multi_put_object_from_file() |
|||
} |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue