Browse Source

add Crypto UserAgent and versionid

master
jojoliang 4 years ago
parent
commit
6ddd32755c
  1. 2
      cos.go
  2. 19
      crypto/crypto_object.go
  3. 18
      crypto/crypto_object_part.go
  4. 155
      crypto/crypto_object_test.go
  5. 2
      crypto/crypto_type.go
  6. 1
      go.mod
  7. 1
      go.sum
  8. 34
      helper.go
  9. 73
      helper_test.go
  10. 34
      object_part.go

2
cos.go

@ -195,9 +195,11 @@ func (c *Client) newRequest(ctx context.Context, baseURL *url.URL, uri, method s
if contentMD5 != "" {
req.Header["Content-MD5"] = []string{contentMD5}
}
if v := req.Header.Get("User-Agent"); v == "" || !strings.HasPrefix(v, userAgent) {
if c.UserAgent != "" {
req.Header.Set("User-Agent", c.UserAgent)
}
}
if req.Header.Get("Content-Type") == "" && contentType != "" {
req.Header.Set("Content-Type", contentType)
}

19
crypto/crypto_object.go

@ -71,6 +71,7 @@ func (s *CryptoObjectService) Put(ctx context.Context, name string, r io.Reader,
opt.XOptionHeader.Add(COSClientSideEncryptionUnencryptedContentLength, strconv.FormatInt(opt.ContentLength, 10))
opt.ContentLength = cc.GetEncryptedLen(opt.ContentLength)
}
opt.XOptionHeader.Add(UserAgent, s.cryptoClient.userAgent)
addCryptoHeaders(opt.XOptionHeader, cc.GetCipherData())
return s.ObjectService.Put(ctx, name, reader, opt)
@ -96,14 +97,14 @@ func (s *CryptoObjectService) PutFromFile(ctx context.Context, name, filePath st
return
}
func (s *CryptoObjectService) Get(ctx context.Context, name string, opt *cos.ObjectGetOptions) (*cos.Response, error) {
meta, err := s.ObjectService.Head(ctx, name, nil)
func (s *CryptoObjectService) Get(ctx context.Context, name string, opt *cos.ObjectGetOptions, id ...string) (*cos.Response, error) {
meta, err := s.ObjectService.Head(ctx, name, nil, id...)
if err != nil {
return meta, err
}
_isEncrypted := isEncrypted(&meta.Header)
if !_isEncrypted {
return s.ObjectService.Get(ctx, name, opt)
return s.ObjectService.Get(ctx, name, opt, id...)
}
envelope := getEnvelopeFromHeader(&meta.Header)
@ -120,6 +121,10 @@ func (s *CryptoObjectService) Get(ctx context.Context, name string, opt *cos.Obj
return nil, fmt.Errorf("get content cipher from envelope failed: %v, object:%v", err, name)
}
opt = cos.CloneObjectGetOptions(opt)
if opt.XOptionHeader == nil {
opt.XOptionHeader = &http.Header{}
}
optRange, err := cos.GetRangeOptions(opt)
if err != nil {
return nil, err
@ -132,7 +137,6 @@ func (s *CryptoObjectService) Get(ctx context.Context, name string, opt *cos.Obj
discardAlignLen = optRange.Start - adjustStart
if discardAlignLen > 0 {
optRange.Start = adjustStart
opt = cos.CloneObjectGetOptions(opt)
opt.Range = cos.FormatRangeOptions(optRange)
}
@ -143,7 +147,8 @@ func (s *CryptoObjectService) Get(ctx context.Context, name string, opt *cos.Obj
return nil, fmt.Errorf("ContentCipher Clone failed:%v, bject:%v", err, name)
}
}
resp, err := s.ObjectService.Get(ctx, name, opt)
opt.XOptionHeader.Add(UserAgent, s.cryptoClient.userAgent)
resp, err := s.ObjectService.Get(ctx, name, opt, id...)
if err != nil {
return resp, err
}
@ -161,8 +166,8 @@ func (s *CryptoObjectService) Get(ctx context.Context, name string, opt *cos.Obj
return resp, err
}
func (s *CryptoObjectService) GetToFile(ctx context.Context, name, localpath string, opt *cos.ObjectGetOptions) (*cos.Response, error) {
resp, err := s.Get(ctx, name, opt)
func (s *CryptoObjectService) GetToFile(ctx context.Context, name, localpath string, opt *cos.ObjectGetOptions, id ...string) (*cos.Response, error) {
resp, err := s.Get(ctx, name, opt, id...)
if err != nil {
return resp, err
}

18
crypto/crypto_object_part.go

@ -44,6 +44,7 @@ func (s *CryptoObjectService) InitiateMultipartUpload(ctx context.Context, name
opt.XOptionHeader.Add(COSClientSideEncryptionDataSize, strconv.FormatInt(cryptoCtx.DataSize, 10))
}
opt.XOptionHeader.Add(COSClientSideEncryptionPartSize, strconv.FormatInt(cryptoCtx.PartSize, 10))
opt.XOptionHeader.Add(UserAgent, s.cryptoClient.userAgent)
addCryptoHeaders(opt.XOptionHeader, contentCipher.GetCipherData())
return s.ObjectService.InitiateMultipartUpload(ctx, name, opt)
@ -54,6 +55,10 @@ func (s *CryptoObjectService) UploadPart(ctx context.Context, name, uploadID str
return nil, fmt.Errorf("CryptoContext's PartSize is zero")
}
opt = cos.CloneObjectUploadPartOptions(opt)
if opt.XOptionHeader == nil {
opt.XOptionHeader = &http.Header{}
}
opt.XOptionHeader.Add(UserAgent, s.cryptoClient.userAgent)
if cryptoCtx.ContentCipher == nil {
return nil, fmt.Errorf("ContentCipher is nil, Please call the InitiateMultipartUpload")
}
@ -77,3 +82,16 @@ func (s *CryptoObjectService) UploadPart(ctx context.Context, name, uploadID str
}
return s.ObjectService.UploadPart(ctx, name, uploadID, partNumber, reader, opt)
}
func (s *CryptoObjectService) CompleteMultipartUpload(ctx context.Context, name, uploadID string, opt *cos.CompleteMultipartUploadOptions) (*cos.CompleteMultipartUploadResult, *cos.Response, error) {
opt = cos.CloneCompleteMultipartUploadOptions(opt)
if opt.XOptionHeader == nil {
opt.XOptionHeader = &http.Header{}
}
opt.XOptionHeader.Add(UserAgent, s.cryptoClient.userAgent)
return s.ObjectService.CompleteMultipartUpload(ctx, name, uploadID, opt)
}
func (s *CryptoObjectService) CopyPart(ctx context.Context, name, uploadID string, partNumber int, sourceURL string, opt *cos.ObjectCopyPartOptions) (*cos.CopyPartResult, *cos.Response, error) {
return nil, nil, fmt.Errorf("CryptoObjectService doesn't support CopyPart")
}

155
crypto/crypto_object_test.go

@ -120,6 +120,45 @@ func (s *CosTestSuite) TestPutGetDeleteObject_Normal_10MB() {
assert.Nil(s.T(), err, "DeleteObject Failed")
}
func (s *CosTestSuite) TestPutGetDeleteObject_VersionID() {
name := "test/objectPut" + time.Now().Format(time.RFC3339)
originData := make([]byte, 1024*1024*10+1)
_, err := rand.Read(originData)
f := bytes.NewReader(originData)
opt := &cos.BucketPutVersionOptions{
Status: "Enabled",
}
_, err = s.CClient.Bucket.PutVersioning(context.Background(), opt)
assert.Nil(s.T(), err, "PutVersioning Failed")
time.Sleep(3 * time.Second)
// 加密存储
resp, err := s.CClient.Object.Put(context.Background(), name, f, nil)
assert.Nil(s.T(), err, "PutObject Failed")
versionId := resp.Header.Get("x-cos-version-id")
_, err = s.CClient.Object.Delete(context.Background(), name)
assert.Nil(s.T(), err, "DeleteObject Failed")
// 解密读取
resp, err = s.CClient.Object.Get(context.Background(), name, nil, versionId)
assert.Nil(s.T(), err, "GetObject Failed")
defer resp.Body.Close()
decryptedData, _ := ioutil.ReadAll(resp.Body)
assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData")
delopt := &cos.ObjectDeleteOptions{
VersionId: versionId,
}
_, err = s.CClient.Object.Delete(context.Background(), name, delopt)
assert.Nil(s.T(), err, "DeleteObject Failed")
opt.Status = "Suspended"
_, err = s.CClient.Bucket.PutVersioning(context.Background(), opt)
assert.Nil(s.T(), err, "PutVersioning Failed")
}
func (s *CosTestSuite) TestPutGetDeleteObject_ZeroFile() {
name := "test/objectPut" + time.Now().Format(time.RFC3339)
// 加密存储
@ -377,6 +416,122 @@ func (s *CosTestSuite) TestPutGetDeleteObject_WithListenerAndRange() {
assert.Nil(s.T(), err, "DeleteObject Failed")
}
func (s *CosTestSuite) TestPutGetDeleteObject_Copy() {
name := "test/objectPut" + time.Now().Format(time.RFC3339)
contentLength := 1024*1024*10 + 1
originData := make([]byte, contentLength)
_, err := rand.Read(originData)
f := bytes.NewReader(originData)
// 加密存储
popt := &cos.ObjectPutOptions{
nil,
&cos.ObjectPutHeaderOptions{
Listener: &cos.DefaultProgressListener{},
},
}
resp, err := s.CClient.Object.Put(context.Background(), name, f, popt)
assert.Nil(s.T(), err, "PutObject Failed")
encryptedDataCRC := resp.Header.Get("x-cos-hash-crc64ecma")
time.Sleep(3 * time.Second)
sourceURL := fmt.Sprintf("%s/%s", s.CClient.BaseURL.BucketURL.Host, name)
{
// x-cos-metadata-directive必须为Copy,否则丢失加密信息,无法解密
dest := "test/ObjectCopy1" + time.Now().Format(time.RFC3339)
res, _, err := s.CClient.Object.Copy(context.Background(), dest, sourceURL, nil)
assert.Nil(s.T(), err, "ObjectCopy Failed")
assert.Equal(s.T(), encryptedDataCRC, res.CRC64, "CRC isn't consistent, return:%v, want:%v", res.CRC64, encryptedDataCRC)
// Range解密读取
for i := 0; i < 3; i++ {
math_rand.Seed(time.Now().UnixNano())
rangeStart := math_rand.Intn(contentLength)
rangeEnd := rangeStart + math_rand.Intn(contentLength-rangeStart)
if rangeEnd == rangeStart || rangeStart >= contentLength-1 {
continue
}
opt := &cos.ObjectGetOptions{
Range: fmt.Sprintf("bytes=%v-%v", rangeStart, rangeEnd),
Listener: &cos.DefaultProgressListener{},
}
resp, err := s.CClient.Object.Get(context.Background(), dest, opt)
assert.Nil(s.T(), err, "GetObject Failed")
defer resp.Body.Close()
decryptedData, _ := ioutil.ReadAll(resp.Body)
assert.Equal(s.T(), bytes.Compare(originData[rangeStart:rangeEnd+1], decryptedData), 0, "decryptData != originData")
}
// 解密读取
resp, err := s.CClient.Object.Get(context.Background(), dest, nil)
assert.Nil(s.T(), err, "GetObject Failed")
defer resp.Body.Close()
decryptedData, _ := ioutil.ReadAll(resp.Body)
assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData")
_, err = s.CClient.Object.Delete(context.Background(), dest)
assert.Nil(s.T(), err, "DeleteObject Failed")
}
{
// x-cos-metadata-directive必须为Copy,否则丢失加密信息,无法解密
opt := &cos.ObjectCopyOptions{
&cos.ObjectCopyHeaderOptions{
XCosMetadataDirective: "Replaced",
},
nil,
}
dest := "test/ObjectCopy2" + time.Now().Format(time.RFC3339)
res, _, err := s.CClient.Object.Copy(context.Background(), dest, sourceURL, opt)
assert.Nil(s.T(), err, "ObjectCopy Failed")
assert.Equal(s.T(), encryptedDataCRC, res.CRC64, "CRC isn't consistent, return:%v, want:%v", res.CRC64, encryptedDataCRC)
// 解密读取
resp, err := s.CClient.Object.Get(context.Background(), dest, nil)
assert.Nil(s.T(), err, "GetObject Failed")
defer resp.Body.Close()
decryptedData, _ := ioutil.ReadAll(resp.Body)
assert.NotEqual(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData")
_, err = s.CClient.Object.Delete(context.Background(), dest)
assert.Nil(s.T(), err, "DeleteObject Failed")
}
{
// MultiCopy若是分块拷贝,则无法拷贝元数据
dest := "test/ObjectCopy3" + time.Now().Format(time.RFC3339)
res, _, err := s.CClient.Object.MultiCopy(context.Background(), dest, sourceURL, nil)
assert.Nil(s.T(), err, "ObjectMultiCopy Failed")
assert.Equal(s.T(), encryptedDataCRC, res.CRC64, "CRC isn't consistent, return:%v, want:%v", res.CRC64, encryptedDataCRC)
// Range解密读取
for i := 0; i < 3; i++ {
math_rand.Seed(time.Now().UnixNano())
rangeStart := math_rand.Intn(contentLength)
rangeEnd := rangeStart + math_rand.Intn(contentLength-rangeStart)
if rangeEnd == rangeStart || rangeStart >= contentLength-1 {
continue
}
opt := &cos.ObjectGetOptions{
Range: fmt.Sprintf("bytes=%v-%v", rangeStart, rangeEnd),
Listener: &cos.DefaultProgressListener{},
}
resp, err := s.CClient.Object.Get(context.Background(), dest, opt)
assert.Nil(s.T(), err, "GetObject Failed")
defer resp.Body.Close()
decryptedData, _ := ioutil.ReadAll(resp.Body)
assert.Equal(s.T(), bytes.Compare(originData[rangeStart:rangeEnd+1], decryptedData), 0, "decryptData != originData")
}
// 解密读取
resp, err := s.CClient.Object.Get(context.Background(), dest, nil)
assert.Nil(s.T(), err, "GetObject Failed")
defer resp.Body.Close()
decryptedData, _ := ioutil.ReadAll(resp.Body)
assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData")
_, err = s.CClient.Object.Delete(context.Background(), dest)
assert.Nil(s.T(), err, "DeleteObject Failed")
}
_, err = s.CClient.Object.Delete(context.Background(), name)
assert.Nil(s.T(), err, "DeleteObject Failed")
}
func TestCosTestSuite(t *testing.T) {
suite.Run(t, new(CosTestSuite))
}

2
crypto/crypto_type.go

@ -19,7 +19,7 @@ const (
COSClientSideEncryptionUnencryptedContentMD5 = "x-cos-meta-client-side-encryption-unencrypted-content-md5"
COSClientSideEncryptionDataSize = "x-cos-meta-client-side-encryption-data-size"
COSClientSideEncryptionPartSize = "x-cos-meta-client-side-encryption-part-size"
COSClientUserAgent = "User-Agent"
UserAgent = "User-Agent"
)
const (

1
go.mod

@ -8,4 +8,5 @@ require (
github.com/google/uuid v1.1.1
github.com/mozillazg/go-httpheader v0.2.1
github.com/stretchr/testify v1.3.0
github.com/tencentcloud/tencentcloud-sdk-go v3.0.233+incompatible
)

1
go.sum

@ -13,3 +13,4 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/tencentcloud/tencentcloud-sdk-go v3.0.233+incompatible/go.mod h1:0PfYow01SHPMhKY31xa+EFz2RStxIqj6JFAJS+IkCi4=

34
helper.go

@ -184,6 +184,19 @@ func CheckReaderLen(reader io.Reader) error {
return errors.New("The single object size you upload can not be larger than 5GB")
}
func cloneHeader(opt *http.Header) *http.Header {
if opt == nil {
return nil
}
h := make(http.Header, len(*opt))
for k, vv := range *opt {
vv2 := make([]string, len(vv))
copy(vv2, vv)
h[k] = vv2
}
return &h
}
func CopyOptionsToMulti(opt *ObjectCopyOptions) *InitiateMultipartUploadOptions {
if opt == nil {
return nil
@ -214,7 +227,6 @@ func CopyOptionsToMulti(opt *ObjectCopyOptions) *InitiateMultipartUploadOptions
return optini
}
// 浅拷贝ObjectPutOptions
func CloneObjectPutOptions(opt *ObjectPutOptions) *ObjectPutOptions {
res := &ObjectPutOptions{
&ACLHeaderOptions{},
@ -226,6 +238,8 @@ func CloneObjectPutOptions(opt *ObjectPutOptions) *ObjectPutOptions {
}
if opt.ObjectPutHeaderOptions != nil {
*res.ObjectPutHeaderOptions = *opt.ObjectPutHeaderOptions
res.XCosMetaXXX = cloneHeader(opt.XCosMetaXXX)
res.XOptionHeader = cloneHeader(opt.XOptionHeader)
}
}
return res
@ -242,16 +256,18 @@ func CloneInitiateMultipartUploadOptions(opt *InitiateMultipartUploadOptions) *I
}
if opt.ObjectPutHeaderOptions != nil {
*res.ObjectPutHeaderOptions = *opt.ObjectPutHeaderOptions
res.XCosMetaXXX = cloneHeader(opt.XCosMetaXXX)
res.XOptionHeader = cloneHeader(opt.XOptionHeader)
}
}
return res
}
// 浅拷贝ObjectUploadPartOptions
func CloneObjectUploadPartOptions(opt *ObjectUploadPartOptions) *ObjectUploadPartOptions {
var res ObjectUploadPartOptions
if opt != nil {
res = *opt
res.XOptionHeader = cloneHeader(opt.XOptionHeader)
}
return &res
}
@ -260,6 +276,20 @@ func CloneObjectGetOptions(opt *ObjectGetOptions) *ObjectGetOptions {
var res ObjectGetOptions
if opt != nil {
res = *opt
res.XOptionHeader = cloneHeader(opt.XOptionHeader)
}
return &res
}
func CloneCompleteMultipartUploadOptions(opt *CompleteMultipartUploadOptions) *CompleteMultipartUploadOptions {
var res CompleteMultipartUploadOptions
if opt != nil {
res.XMLName = opt.XMLName
if len(opt.Parts) > 0 {
res.Parts = make([]Object, len(opt.Parts))
copy(res.Parts, opt.Parts)
}
res.XOptionHeader = cloneHeader(opt.XOptionHeader)
}
return &res
}

73
helper_test.go

@ -1,7 +1,10 @@
package cos
import (
"encoding/xml"
"fmt"
"net/http"
"reflect"
"testing"
)
@ -22,3 +25,73 @@ func Test_calMD5Digest(t *testing.T) {
t.Errorf("calMD5Digest request md5: %+v, want %+v", got, want)
}
}
func Test_cloneHeader(t *testing.T) {
ori := http.Header{}
opt := &ori
opt.Add("TestHeader1", "h1")
opt.Add("TestHeader1", "h2")
res := cloneHeader(opt)
if !reflect.DeepEqual(res, opt) {
t.Errorf("cloneHeader, returned:%+v, want:%+v", res, opt)
}
if !reflect.DeepEqual(ori, *opt) {
t.Errorf("cloneHeader, returned:%+v, want:%+v", *opt, ori)
}
res.Add("cloneHeader1", "c1")
res.Add("cloneHeader2", "c2")
if v := opt.Get("cloneHeader1"); v != "" {
t.Errorf("cloneHeader, returned:%+v, want:%+v", res, opt)
}
if v := opt.Get("cloneHeader2"); v != "" {
t.Errorf("cloneHeader, returned:%+v, want:%+v", res, opt)
}
opt = &http.Header{}
res = cloneHeader(opt)
if !reflect.DeepEqual(res, opt) {
t.Errorf("cloneHeader, returned:%+v, want:%+v", res, opt)
}
}
func Test_CloneCompleteMultipartUploadOptions(t *testing.T) {
ori := CompleteMultipartUploadOptions{
XMLName: xml.Name{Local: "CompleteMultipartUploadResult"},
Parts: []Object{
{
Key: "Key1",
ETag: "Etag1",
},
{
Key: "Key2",
ETag: "Etag2",
},
},
XOptionHeader: &http.Header{},
}
ori.XOptionHeader.Add("Test", "value")
opt := &ori
res := CloneCompleteMultipartUploadOptions(opt)
if !reflect.DeepEqual(res, opt) {
t.Errorf("CloneCompleteMultipartUploadOptions, returned:%+v,want:%+v", res, opt)
}
if !reflect.DeepEqual(ori, *opt) {
t.Errorf("CloneCompleteMultipartUploadOptions, returned:%+v,want:%+v", *opt, ori)
}
res.XOptionHeader.Add("TestClone", "value")
if v := opt.XOptionHeader.Get("TestClone"); v != "" {
t.Errorf("CloneCompleteMultipartUploadOptions, returned:%+v,want:%+v", res, opt)
}
opt = &CompleteMultipartUploadOptions{}
res = CloneCompleteMultipartUploadOptions(opt)
if !reflect.DeepEqual(res, opt) {
t.Errorf("CloneCompleteMultipartUploadOptions, returned:%+v,want:%+v", res, opt)
}
res.Parts = append(res.Parts, Object{Key: "K", ETag: "T"})
if len(opt.Parts) > 0 {
t.Errorf("CloneCompleteMultipartUploadOptions Failed")
}
if reflect.DeepEqual(res, opt) {
t.Errorf("CloneCompleteMultipartUploadOptions, returned:%+v,want:%+v", res, opt)
}
}

34
object_part.go

@ -402,37 +402,7 @@ func (s *ObjectService) innerHead(ctx context.Context, sourceURL string, opt *Ob
return
}
func SplitCopyFileIntoChunks(totalBytes int64, partSize int64) ([]Chunk, int, error) {
var partNum int64
if partSize > 0 {
partSize = partSize * 1024 * 1024
partNum = totalBytes / partSize
if partNum >= 10000 {
return nil, 0, errors.New("Too many parts, out of 10000")
}
} else {
partNum, partSize = DividePart(totalBytes, 128)
}
var chunks []Chunk
var chunk = Chunk{}
for i := int64(0); i < partNum; i++ {
chunk.Number = int(i + 1)
chunk.OffSet = i * partSize
chunk.Size = partSize
chunks = append(chunks, chunk)
}
if totalBytes%partSize > 0 {
chunk.Number = len(chunks) + 1
chunk.OffSet = int64(len(chunks)) * partSize
chunk.Size = totalBytes % partSize
chunks = append(chunks, chunk)
partNum++
}
return chunks, int(partNum), nil
}
// 如果源对象大于5G,则采用分块复制的方式进行拷贝,此时源对象的元信息如果COPY
func (s *ObjectService) MultiCopy(ctx context.Context, name string, sourceURL string, opt *MultiCopyOptions, id ...string) (*ObjectCopyResult, *Response, error) {
resp, err := s.innerHead(ctx, sourceURL, nil, id)
if err != nil {
@ -455,7 +425,7 @@ func (s *ObjectService) MultiCopy(ctx context.Context, name string, sourceURL st
if opt == nil {
opt = &MultiCopyOptions{}
}
chunks, partNum, err := SplitCopyFileIntoChunks(totalBytes, opt.PartSize)
chunks, partNum, err := SplitSizeIntoChunks(totalBytes, opt.PartSize*1024*1024)
if err != nil {
return nil, nil, err
}

Loading…
Cancel
Save