add Crypto UserAgent and versionid
This commit is contained in:
34
cos.go
34
cos.go
@@ -134,23 +134,23 @@ func NewClient(uri *BaseURL, httpClient *http.Client) *Client {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type Credential struct {
|
type Credential struct {
|
||||||
SecretID string
|
SecretID string
|
||||||
SecretKey string
|
SecretKey string
|
||||||
SessionToken string
|
SessionToken string
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Client) GetCredential() *Credential {
|
func (c *Client) GetCredential() *Credential {
|
||||||
auth, ok := c.client.Transport.(*AuthorizationTransport)
|
auth, ok := c.client.Transport.(*AuthorizationTransport)
|
||||||
if !ok {
|
if !ok {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
auth.rwLocker.Lock()
|
auth.rwLocker.Lock()
|
||||||
defer auth.rwLocker.Unlock()
|
defer auth.rwLocker.Unlock()
|
||||||
return &Credential{
|
return &Credential{
|
||||||
SecretID: auth.SecretID,
|
SecretID: auth.SecretID,
|
||||||
SecretKey: auth.SecretKey,
|
SecretKey: auth.SecretKey,
|
||||||
SessionToken: auth.SessionToken,
|
SessionToken: auth.SessionToken,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Client) newRequest(ctx context.Context, baseURL *url.URL, uri, method string, body interface{}, optQuery interface{}, optHeader interface{}) (req *http.Request, err error) {
|
func (c *Client) newRequest(ctx context.Context, baseURL *url.URL, uri, method string, body interface{}, optQuery interface{}, optHeader interface{}) (req *http.Request, err error) {
|
||||||
@@ -195,8 +195,10 @@ func (c *Client) newRequest(ctx context.Context, baseURL *url.URL, uri, method s
|
|||||||
if contentMD5 != "" {
|
if contentMD5 != "" {
|
||||||
req.Header["Content-MD5"] = []string{contentMD5}
|
req.Header["Content-MD5"] = []string{contentMD5}
|
||||||
}
|
}
|
||||||
if c.UserAgent != "" {
|
if v := req.Header.Get("User-Agent"); v == "" || !strings.HasPrefix(v, userAgent) {
|
||||||
req.Header.Set("User-Agent", c.UserAgent)
|
if c.UserAgent != "" {
|
||||||
|
req.Header.Set("User-Agent", c.UserAgent)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if req.Header.Get("Content-Type") == "" && contentType != "" {
|
if req.Header.Get("Content-Type") == "" && contentType != "" {
|
||||||
req.Header.Set("Content-Type", contentType)
|
req.Header.Set("Content-Type", contentType)
|
||||||
|
|||||||
@@ -71,6 +71,7 @@ func (s *CryptoObjectService) Put(ctx context.Context, name string, r io.Reader,
|
|||||||
opt.XOptionHeader.Add(COSClientSideEncryptionUnencryptedContentLength, strconv.FormatInt(opt.ContentLength, 10))
|
opt.XOptionHeader.Add(COSClientSideEncryptionUnencryptedContentLength, strconv.FormatInt(opt.ContentLength, 10))
|
||||||
opt.ContentLength = cc.GetEncryptedLen(opt.ContentLength)
|
opt.ContentLength = cc.GetEncryptedLen(opt.ContentLength)
|
||||||
}
|
}
|
||||||
|
opt.XOptionHeader.Add(UserAgent, s.cryptoClient.userAgent)
|
||||||
addCryptoHeaders(opt.XOptionHeader, cc.GetCipherData())
|
addCryptoHeaders(opt.XOptionHeader, cc.GetCipherData())
|
||||||
|
|
||||||
return s.ObjectService.Put(ctx, name, reader, opt)
|
return s.ObjectService.Put(ctx, name, reader, opt)
|
||||||
@@ -96,14 +97,14 @@ func (s *CryptoObjectService) PutFromFile(ctx context.Context, name, filePath st
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *CryptoObjectService) Get(ctx context.Context, name string, opt *cos.ObjectGetOptions) (*cos.Response, error) {
|
func (s *CryptoObjectService) Get(ctx context.Context, name string, opt *cos.ObjectGetOptions, id ...string) (*cos.Response, error) {
|
||||||
meta, err := s.ObjectService.Head(ctx, name, nil)
|
meta, err := s.ObjectService.Head(ctx, name, nil, id...)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return meta, err
|
return meta, err
|
||||||
}
|
}
|
||||||
_isEncrypted := isEncrypted(&meta.Header)
|
_isEncrypted := isEncrypted(&meta.Header)
|
||||||
if !_isEncrypted {
|
if !_isEncrypted {
|
||||||
return s.ObjectService.Get(ctx, name, opt)
|
return s.ObjectService.Get(ctx, name, opt, id...)
|
||||||
}
|
}
|
||||||
|
|
||||||
envelope := getEnvelopeFromHeader(&meta.Header)
|
envelope := getEnvelopeFromHeader(&meta.Header)
|
||||||
@@ -120,6 +121,10 @@ func (s *CryptoObjectService) Get(ctx context.Context, name string, opt *cos.Obj
|
|||||||
return nil, fmt.Errorf("get content cipher from envelope failed: %v, object:%v", err, name)
|
return nil, fmt.Errorf("get content cipher from envelope failed: %v, object:%v", err, name)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
opt = cos.CloneObjectGetOptions(opt)
|
||||||
|
if opt.XOptionHeader == nil {
|
||||||
|
opt.XOptionHeader = &http.Header{}
|
||||||
|
}
|
||||||
optRange, err := cos.GetRangeOptions(opt)
|
optRange, err := cos.GetRangeOptions(opt)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -132,7 +137,6 @@ func (s *CryptoObjectService) Get(ctx context.Context, name string, opt *cos.Obj
|
|||||||
discardAlignLen = optRange.Start - adjustStart
|
discardAlignLen = optRange.Start - adjustStart
|
||||||
if discardAlignLen > 0 {
|
if discardAlignLen > 0 {
|
||||||
optRange.Start = adjustStart
|
optRange.Start = adjustStart
|
||||||
opt = cos.CloneObjectGetOptions(opt)
|
|
||||||
opt.Range = cos.FormatRangeOptions(optRange)
|
opt.Range = cos.FormatRangeOptions(optRange)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -143,7 +147,8 @@ func (s *CryptoObjectService) Get(ctx context.Context, name string, opt *cos.Obj
|
|||||||
return nil, fmt.Errorf("ContentCipher Clone failed:%v, bject:%v", err, name)
|
return nil, fmt.Errorf("ContentCipher Clone failed:%v, bject:%v", err, name)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
resp, err := s.ObjectService.Get(ctx, name, opt)
|
opt.XOptionHeader.Add(UserAgent, s.cryptoClient.userAgent)
|
||||||
|
resp, err := s.ObjectService.Get(ctx, name, opt, id...)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return resp, err
|
return resp, err
|
||||||
}
|
}
|
||||||
@@ -161,8 +166,8 @@ func (s *CryptoObjectService) Get(ctx context.Context, name string, opt *cos.Obj
|
|||||||
return resp, err
|
return resp, err
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *CryptoObjectService) GetToFile(ctx context.Context, name, localpath string, opt *cos.ObjectGetOptions) (*cos.Response, error) {
|
func (s *CryptoObjectService) GetToFile(ctx context.Context, name, localpath string, opt *cos.ObjectGetOptions, id ...string) (*cos.Response, error) {
|
||||||
resp, err := s.Get(ctx, name, opt)
|
resp, err := s.Get(ctx, name, opt, id...)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return resp, err
|
return resp, err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -44,6 +44,7 @@ func (s *CryptoObjectService) InitiateMultipartUpload(ctx context.Context, name
|
|||||||
opt.XOptionHeader.Add(COSClientSideEncryptionDataSize, strconv.FormatInt(cryptoCtx.DataSize, 10))
|
opt.XOptionHeader.Add(COSClientSideEncryptionDataSize, strconv.FormatInt(cryptoCtx.DataSize, 10))
|
||||||
}
|
}
|
||||||
opt.XOptionHeader.Add(COSClientSideEncryptionPartSize, strconv.FormatInt(cryptoCtx.PartSize, 10))
|
opt.XOptionHeader.Add(COSClientSideEncryptionPartSize, strconv.FormatInt(cryptoCtx.PartSize, 10))
|
||||||
|
opt.XOptionHeader.Add(UserAgent, s.cryptoClient.userAgent)
|
||||||
addCryptoHeaders(opt.XOptionHeader, contentCipher.GetCipherData())
|
addCryptoHeaders(opt.XOptionHeader, contentCipher.GetCipherData())
|
||||||
|
|
||||||
return s.ObjectService.InitiateMultipartUpload(ctx, name, opt)
|
return s.ObjectService.InitiateMultipartUpload(ctx, name, opt)
|
||||||
@@ -54,6 +55,10 @@ func (s *CryptoObjectService) UploadPart(ctx context.Context, name, uploadID str
|
|||||||
return nil, fmt.Errorf("CryptoContext's PartSize is zero")
|
return nil, fmt.Errorf("CryptoContext's PartSize is zero")
|
||||||
}
|
}
|
||||||
opt = cos.CloneObjectUploadPartOptions(opt)
|
opt = cos.CloneObjectUploadPartOptions(opt)
|
||||||
|
if opt.XOptionHeader == nil {
|
||||||
|
opt.XOptionHeader = &http.Header{}
|
||||||
|
}
|
||||||
|
opt.XOptionHeader.Add(UserAgent, s.cryptoClient.userAgent)
|
||||||
if cryptoCtx.ContentCipher == nil {
|
if cryptoCtx.ContentCipher == nil {
|
||||||
return nil, fmt.Errorf("ContentCipher is nil, Please call the InitiateMultipartUpload")
|
return nil, fmt.Errorf("ContentCipher is nil, Please call the InitiateMultipartUpload")
|
||||||
}
|
}
|
||||||
@@ -77,3 +82,16 @@ func (s *CryptoObjectService) UploadPart(ctx context.Context, name, uploadID str
|
|||||||
}
|
}
|
||||||
return s.ObjectService.UploadPart(ctx, name, uploadID, partNumber, reader, opt)
|
return s.ObjectService.UploadPart(ctx, name, uploadID, partNumber, reader, opt)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *CryptoObjectService) CompleteMultipartUpload(ctx context.Context, name, uploadID string, opt *cos.CompleteMultipartUploadOptions) (*cos.CompleteMultipartUploadResult, *cos.Response, error) {
|
||||||
|
opt = cos.CloneCompleteMultipartUploadOptions(opt)
|
||||||
|
if opt.XOptionHeader == nil {
|
||||||
|
opt.XOptionHeader = &http.Header{}
|
||||||
|
}
|
||||||
|
opt.XOptionHeader.Add(UserAgent, s.cryptoClient.userAgent)
|
||||||
|
return s.ObjectService.CompleteMultipartUpload(ctx, name, uploadID, opt)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *CryptoObjectService) CopyPart(ctx context.Context, name, uploadID string, partNumber int, sourceURL string, opt *cos.ObjectCopyPartOptions) (*cos.CopyPartResult, *cos.Response, error) {
|
||||||
|
return nil, nil, fmt.Errorf("CryptoObjectService doesn't support CopyPart")
|
||||||
|
}
|
||||||
|
|||||||
@@ -120,6 +120,45 @@ func (s *CosTestSuite) TestPutGetDeleteObject_Normal_10MB() {
|
|||||||
assert.Nil(s.T(), err, "DeleteObject Failed")
|
assert.Nil(s.T(), err, "DeleteObject Failed")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *CosTestSuite) TestPutGetDeleteObject_VersionID() {
|
||||||
|
name := "test/objectPut" + time.Now().Format(time.RFC3339)
|
||||||
|
originData := make([]byte, 1024*1024*10+1)
|
||||||
|
_, err := rand.Read(originData)
|
||||||
|
f := bytes.NewReader(originData)
|
||||||
|
|
||||||
|
opt := &cos.BucketPutVersionOptions{
|
||||||
|
Status: "Enabled",
|
||||||
|
}
|
||||||
|
_, err = s.CClient.Bucket.PutVersioning(context.Background(), opt)
|
||||||
|
assert.Nil(s.T(), err, "PutVersioning Failed")
|
||||||
|
time.Sleep(3 * time.Second)
|
||||||
|
|
||||||
|
// 加密存储
|
||||||
|
resp, err := s.CClient.Object.Put(context.Background(), name, f, nil)
|
||||||
|
assert.Nil(s.T(), err, "PutObject Failed")
|
||||||
|
versionId := resp.Header.Get("x-cos-version-id")
|
||||||
|
|
||||||
|
_, err = s.CClient.Object.Delete(context.Background(), name)
|
||||||
|
assert.Nil(s.T(), err, "DeleteObject Failed")
|
||||||
|
|
||||||
|
// 解密读取
|
||||||
|
resp, err = s.CClient.Object.Get(context.Background(), name, nil, versionId)
|
||||||
|
assert.Nil(s.T(), err, "GetObject Failed")
|
||||||
|
defer resp.Body.Close()
|
||||||
|
decryptedData, _ := ioutil.ReadAll(resp.Body)
|
||||||
|
assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData")
|
||||||
|
|
||||||
|
delopt := &cos.ObjectDeleteOptions{
|
||||||
|
VersionId: versionId,
|
||||||
|
}
|
||||||
|
_, err = s.CClient.Object.Delete(context.Background(), name, delopt)
|
||||||
|
assert.Nil(s.T(), err, "DeleteObject Failed")
|
||||||
|
|
||||||
|
opt.Status = "Suspended"
|
||||||
|
_, err = s.CClient.Bucket.PutVersioning(context.Background(), opt)
|
||||||
|
assert.Nil(s.T(), err, "PutVersioning Failed")
|
||||||
|
}
|
||||||
|
|
||||||
func (s *CosTestSuite) TestPutGetDeleteObject_ZeroFile() {
|
func (s *CosTestSuite) TestPutGetDeleteObject_ZeroFile() {
|
||||||
name := "test/objectPut" + time.Now().Format(time.RFC3339)
|
name := "test/objectPut" + time.Now().Format(time.RFC3339)
|
||||||
// 加密存储
|
// 加密存储
|
||||||
@@ -377,6 +416,122 @@ func (s *CosTestSuite) TestPutGetDeleteObject_WithListenerAndRange() {
|
|||||||
assert.Nil(s.T(), err, "DeleteObject Failed")
|
assert.Nil(s.T(), err, "DeleteObject Failed")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *CosTestSuite) TestPutGetDeleteObject_Copy() {
|
||||||
|
name := "test/objectPut" + time.Now().Format(time.RFC3339)
|
||||||
|
contentLength := 1024*1024*10 + 1
|
||||||
|
originData := make([]byte, contentLength)
|
||||||
|
_, err := rand.Read(originData)
|
||||||
|
f := bytes.NewReader(originData)
|
||||||
|
|
||||||
|
// 加密存储
|
||||||
|
popt := &cos.ObjectPutOptions{
|
||||||
|
nil,
|
||||||
|
&cos.ObjectPutHeaderOptions{
|
||||||
|
Listener: &cos.DefaultProgressListener{},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
resp, err := s.CClient.Object.Put(context.Background(), name, f, popt)
|
||||||
|
assert.Nil(s.T(), err, "PutObject Failed")
|
||||||
|
encryptedDataCRC := resp.Header.Get("x-cos-hash-crc64ecma")
|
||||||
|
time.Sleep(3 * time.Second)
|
||||||
|
sourceURL := fmt.Sprintf("%s/%s", s.CClient.BaseURL.BucketURL.Host, name)
|
||||||
|
{
|
||||||
|
// x-cos-metadata-directive必须为Copy,否则丢失加密信息,无法解密
|
||||||
|
dest := "test/ObjectCopy1" + time.Now().Format(time.RFC3339)
|
||||||
|
res, _, err := s.CClient.Object.Copy(context.Background(), dest, sourceURL, nil)
|
||||||
|
assert.Nil(s.T(), err, "ObjectCopy Failed")
|
||||||
|
assert.Equal(s.T(), encryptedDataCRC, res.CRC64, "CRC isn't consistent, return:%v, want:%v", res.CRC64, encryptedDataCRC)
|
||||||
|
|
||||||
|
// Range解密读取
|
||||||
|
for i := 0; i < 3; i++ {
|
||||||
|
math_rand.Seed(time.Now().UnixNano())
|
||||||
|
rangeStart := math_rand.Intn(contentLength)
|
||||||
|
rangeEnd := rangeStart + math_rand.Intn(contentLength-rangeStart)
|
||||||
|
if rangeEnd == rangeStart || rangeStart >= contentLength-1 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
opt := &cos.ObjectGetOptions{
|
||||||
|
Range: fmt.Sprintf("bytes=%v-%v", rangeStart, rangeEnd),
|
||||||
|
Listener: &cos.DefaultProgressListener{},
|
||||||
|
}
|
||||||
|
resp, err := s.CClient.Object.Get(context.Background(), dest, opt)
|
||||||
|
assert.Nil(s.T(), err, "GetObject Failed")
|
||||||
|
defer resp.Body.Close()
|
||||||
|
decryptedData, _ := ioutil.ReadAll(resp.Body)
|
||||||
|
assert.Equal(s.T(), bytes.Compare(originData[rangeStart:rangeEnd+1], decryptedData), 0, "decryptData != originData")
|
||||||
|
}
|
||||||
|
// 解密读取
|
||||||
|
resp, err := s.CClient.Object.Get(context.Background(), dest, nil)
|
||||||
|
assert.Nil(s.T(), err, "GetObject Failed")
|
||||||
|
defer resp.Body.Close()
|
||||||
|
decryptedData, _ := ioutil.ReadAll(resp.Body)
|
||||||
|
assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData")
|
||||||
|
|
||||||
|
_, err = s.CClient.Object.Delete(context.Background(), dest)
|
||||||
|
assert.Nil(s.T(), err, "DeleteObject Failed")
|
||||||
|
}
|
||||||
|
{
|
||||||
|
// x-cos-metadata-directive必须为Copy,否则丢失加密信息,无法解密
|
||||||
|
opt := &cos.ObjectCopyOptions{
|
||||||
|
&cos.ObjectCopyHeaderOptions{
|
||||||
|
XCosMetadataDirective: "Replaced",
|
||||||
|
},
|
||||||
|
nil,
|
||||||
|
}
|
||||||
|
dest := "test/ObjectCopy2" + time.Now().Format(time.RFC3339)
|
||||||
|
res, _, err := s.CClient.Object.Copy(context.Background(), dest, sourceURL, opt)
|
||||||
|
assert.Nil(s.T(), err, "ObjectCopy Failed")
|
||||||
|
assert.Equal(s.T(), encryptedDataCRC, res.CRC64, "CRC isn't consistent, return:%v, want:%v", res.CRC64, encryptedDataCRC)
|
||||||
|
|
||||||
|
// 解密读取
|
||||||
|
resp, err := s.CClient.Object.Get(context.Background(), dest, nil)
|
||||||
|
assert.Nil(s.T(), err, "GetObject Failed")
|
||||||
|
defer resp.Body.Close()
|
||||||
|
decryptedData, _ := ioutil.ReadAll(resp.Body)
|
||||||
|
assert.NotEqual(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData")
|
||||||
|
|
||||||
|
_, err = s.CClient.Object.Delete(context.Background(), dest)
|
||||||
|
assert.Nil(s.T(), err, "DeleteObject Failed")
|
||||||
|
}
|
||||||
|
{
|
||||||
|
// MultiCopy若是分块拷贝,则无法拷贝元数据
|
||||||
|
dest := "test/ObjectCopy3" + time.Now().Format(time.RFC3339)
|
||||||
|
res, _, err := s.CClient.Object.MultiCopy(context.Background(), dest, sourceURL, nil)
|
||||||
|
assert.Nil(s.T(), err, "ObjectMultiCopy Failed")
|
||||||
|
assert.Equal(s.T(), encryptedDataCRC, res.CRC64, "CRC isn't consistent, return:%v, want:%v", res.CRC64, encryptedDataCRC)
|
||||||
|
// Range解密读取
|
||||||
|
for i := 0; i < 3; i++ {
|
||||||
|
math_rand.Seed(time.Now().UnixNano())
|
||||||
|
rangeStart := math_rand.Intn(contentLength)
|
||||||
|
rangeEnd := rangeStart + math_rand.Intn(contentLength-rangeStart)
|
||||||
|
if rangeEnd == rangeStart || rangeStart >= contentLength-1 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
opt := &cos.ObjectGetOptions{
|
||||||
|
Range: fmt.Sprintf("bytes=%v-%v", rangeStart, rangeEnd),
|
||||||
|
Listener: &cos.DefaultProgressListener{},
|
||||||
|
}
|
||||||
|
resp, err := s.CClient.Object.Get(context.Background(), dest, opt)
|
||||||
|
assert.Nil(s.T(), err, "GetObject Failed")
|
||||||
|
defer resp.Body.Close()
|
||||||
|
decryptedData, _ := ioutil.ReadAll(resp.Body)
|
||||||
|
assert.Equal(s.T(), bytes.Compare(originData[rangeStart:rangeEnd+1], decryptedData), 0, "decryptData != originData")
|
||||||
|
}
|
||||||
|
// 解密读取
|
||||||
|
resp, err := s.CClient.Object.Get(context.Background(), dest, nil)
|
||||||
|
assert.Nil(s.T(), err, "GetObject Failed")
|
||||||
|
defer resp.Body.Close()
|
||||||
|
decryptedData, _ := ioutil.ReadAll(resp.Body)
|
||||||
|
assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData")
|
||||||
|
|
||||||
|
_, err = s.CClient.Object.Delete(context.Background(), dest)
|
||||||
|
assert.Nil(s.T(), err, "DeleteObject Failed")
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = s.CClient.Object.Delete(context.Background(), name)
|
||||||
|
assert.Nil(s.T(), err, "DeleteObject Failed")
|
||||||
|
}
|
||||||
|
|
||||||
func TestCosTestSuite(t *testing.T) {
|
func TestCosTestSuite(t *testing.T) {
|
||||||
suite.Run(t, new(CosTestSuite))
|
suite.Run(t, new(CosTestSuite))
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ const (
|
|||||||
COSClientSideEncryptionUnencryptedContentMD5 = "x-cos-meta-client-side-encryption-unencrypted-content-md5"
|
COSClientSideEncryptionUnencryptedContentMD5 = "x-cos-meta-client-side-encryption-unencrypted-content-md5"
|
||||||
COSClientSideEncryptionDataSize = "x-cos-meta-client-side-encryption-data-size"
|
COSClientSideEncryptionDataSize = "x-cos-meta-client-side-encryption-data-size"
|
||||||
COSClientSideEncryptionPartSize = "x-cos-meta-client-side-encryption-part-size"
|
COSClientSideEncryptionPartSize = "x-cos-meta-client-side-encryption-part-size"
|
||||||
COSClientUserAgent = "User-Agent"
|
UserAgent = "User-Agent"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
|||||||
1
go.mod
1
go.mod
@@ -8,4 +8,5 @@ require (
|
|||||||
github.com/google/uuid v1.1.1
|
github.com/google/uuid v1.1.1
|
||||||
github.com/mozillazg/go-httpheader v0.2.1
|
github.com/mozillazg/go-httpheader v0.2.1
|
||||||
github.com/stretchr/testify v1.3.0
|
github.com/stretchr/testify v1.3.0
|
||||||
|
github.com/tencentcloud/tencentcloud-sdk-go v3.0.233+incompatible
|
||||||
)
|
)
|
||||||
|
|||||||
1
go.sum
1
go.sum
@@ -13,3 +13,4 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
|
|||||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||||
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
|
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
|
||||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||||
|
github.com/tencentcloud/tencentcloud-sdk-go v3.0.233+incompatible/go.mod h1:0PfYow01SHPMhKY31xa+EFz2RStxIqj6JFAJS+IkCi4=
|
||||||
|
|||||||
34
helper.go
34
helper.go
@@ -184,6 +184,19 @@ func CheckReaderLen(reader io.Reader) error {
|
|||||||
return errors.New("The single object size you upload can not be larger than 5GB")
|
return errors.New("The single object size you upload can not be larger than 5GB")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func cloneHeader(opt *http.Header) *http.Header {
|
||||||
|
if opt == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
h := make(http.Header, len(*opt))
|
||||||
|
for k, vv := range *opt {
|
||||||
|
vv2 := make([]string, len(vv))
|
||||||
|
copy(vv2, vv)
|
||||||
|
h[k] = vv2
|
||||||
|
}
|
||||||
|
return &h
|
||||||
|
}
|
||||||
|
|
||||||
func CopyOptionsToMulti(opt *ObjectCopyOptions) *InitiateMultipartUploadOptions {
|
func CopyOptionsToMulti(opt *ObjectCopyOptions) *InitiateMultipartUploadOptions {
|
||||||
if opt == nil {
|
if opt == nil {
|
||||||
return nil
|
return nil
|
||||||
@@ -214,7 +227,6 @@ func CopyOptionsToMulti(opt *ObjectCopyOptions) *InitiateMultipartUploadOptions
|
|||||||
return optini
|
return optini
|
||||||
}
|
}
|
||||||
|
|
||||||
// 浅拷贝ObjectPutOptions
|
|
||||||
func CloneObjectPutOptions(opt *ObjectPutOptions) *ObjectPutOptions {
|
func CloneObjectPutOptions(opt *ObjectPutOptions) *ObjectPutOptions {
|
||||||
res := &ObjectPutOptions{
|
res := &ObjectPutOptions{
|
||||||
&ACLHeaderOptions{},
|
&ACLHeaderOptions{},
|
||||||
@@ -226,6 +238,8 @@ func CloneObjectPutOptions(opt *ObjectPutOptions) *ObjectPutOptions {
|
|||||||
}
|
}
|
||||||
if opt.ObjectPutHeaderOptions != nil {
|
if opt.ObjectPutHeaderOptions != nil {
|
||||||
*res.ObjectPutHeaderOptions = *opt.ObjectPutHeaderOptions
|
*res.ObjectPutHeaderOptions = *opt.ObjectPutHeaderOptions
|
||||||
|
res.XCosMetaXXX = cloneHeader(opt.XCosMetaXXX)
|
||||||
|
res.XOptionHeader = cloneHeader(opt.XOptionHeader)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return res
|
return res
|
||||||
@@ -242,16 +256,18 @@ func CloneInitiateMultipartUploadOptions(opt *InitiateMultipartUploadOptions) *I
|
|||||||
}
|
}
|
||||||
if opt.ObjectPutHeaderOptions != nil {
|
if opt.ObjectPutHeaderOptions != nil {
|
||||||
*res.ObjectPutHeaderOptions = *opt.ObjectPutHeaderOptions
|
*res.ObjectPutHeaderOptions = *opt.ObjectPutHeaderOptions
|
||||||
|
res.XCosMetaXXX = cloneHeader(opt.XCosMetaXXX)
|
||||||
|
res.XOptionHeader = cloneHeader(opt.XOptionHeader)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return res
|
return res
|
||||||
}
|
}
|
||||||
|
|
||||||
// 浅拷贝ObjectUploadPartOptions
|
|
||||||
func CloneObjectUploadPartOptions(opt *ObjectUploadPartOptions) *ObjectUploadPartOptions {
|
func CloneObjectUploadPartOptions(opt *ObjectUploadPartOptions) *ObjectUploadPartOptions {
|
||||||
var res ObjectUploadPartOptions
|
var res ObjectUploadPartOptions
|
||||||
if opt != nil {
|
if opt != nil {
|
||||||
res = *opt
|
res = *opt
|
||||||
|
res.XOptionHeader = cloneHeader(opt.XOptionHeader)
|
||||||
}
|
}
|
||||||
return &res
|
return &res
|
||||||
}
|
}
|
||||||
@@ -260,10 +276,24 @@ func CloneObjectGetOptions(opt *ObjectGetOptions) *ObjectGetOptions {
|
|||||||
var res ObjectGetOptions
|
var res ObjectGetOptions
|
||||||
if opt != nil {
|
if opt != nil {
|
||||||
res = *opt
|
res = *opt
|
||||||
|
res.XOptionHeader = cloneHeader(opt.XOptionHeader)
|
||||||
}
|
}
|
||||||
return &res
|
return &res
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func CloneCompleteMultipartUploadOptions(opt *CompleteMultipartUploadOptions) *CompleteMultipartUploadOptions {
|
||||||
|
var res CompleteMultipartUploadOptions
|
||||||
|
if opt != nil {
|
||||||
|
res.XMLName = opt.XMLName
|
||||||
|
if len(opt.Parts) > 0 {
|
||||||
|
res.Parts = make([]Object, len(opt.Parts))
|
||||||
|
copy(res.Parts, opt.Parts)
|
||||||
|
}
|
||||||
|
res.XOptionHeader = cloneHeader(opt.XOptionHeader)
|
||||||
|
}
|
||||||
|
return &res
|
||||||
|
}
|
||||||
|
|
||||||
type RangeOptions struct {
|
type RangeOptions struct {
|
||||||
HasStart bool
|
HasStart bool
|
||||||
HasEnd bool
|
HasEnd bool
|
||||||
|
|||||||
@@ -1,7 +1,10 @@
|
|||||||
package cos
|
package cos
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"encoding/xml"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"reflect"
|
||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -22,3 +25,73 @@ func Test_calMD5Digest(t *testing.T) {
|
|||||||
t.Errorf("calMD5Digest request md5: %+v, want %+v", got, want)
|
t.Errorf("calMD5Digest request md5: %+v, want %+v", got, want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func Test_cloneHeader(t *testing.T) {
|
||||||
|
ori := http.Header{}
|
||||||
|
opt := &ori
|
||||||
|
opt.Add("TestHeader1", "h1")
|
||||||
|
opt.Add("TestHeader1", "h2")
|
||||||
|
res := cloneHeader(opt)
|
||||||
|
if !reflect.DeepEqual(res, opt) {
|
||||||
|
t.Errorf("cloneHeader, returned:%+v, want:%+v", res, opt)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(ori, *opt) {
|
||||||
|
t.Errorf("cloneHeader, returned:%+v, want:%+v", *opt, ori)
|
||||||
|
}
|
||||||
|
res.Add("cloneHeader1", "c1")
|
||||||
|
res.Add("cloneHeader2", "c2")
|
||||||
|
if v := opt.Get("cloneHeader1"); v != "" {
|
||||||
|
t.Errorf("cloneHeader, returned:%+v, want:%+v", res, opt)
|
||||||
|
}
|
||||||
|
if v := opt.Get("cloneHeader2"); v != "" {
|
||||||
|
t.Errorf("cloneHeader, returned:%+v, want:%+v", res, opt)
|
||||||
|
}
|
||||||
|
opt = &http.Header{}
|
||||||
|
res = cloneHeader(opt)
|
||||||
|
if !reflect.DeepEqual(res, opt) {
|
||||||
|
t.Errorf("cloneHeader, returned:%+v, want:%+v", res, opt)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func Test_CloneCompleteMultipartUploadOptions(t *testing.T) {
|
||||||
|
ori := CompleteMultipartUploadOptions{
|
||||||
|
XMLName: xml.Name{Local: "CompleteMultipartUploadResult"},
|
||||||
|
Parts: []Object{
|
||||||
|
{
|
||||||
|
Key: "Key1",
|
||||||
|
ETag: "Etag1",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Key: "Key2",
|
||||||
|
ETag: "Etag2",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
XOptionHeader: &http.Header{},
|
||||||
|
}
|
||||||
|
ori.XOptionHeader.Add("Test", "value")
|
||||||
|
opt := &ori
|
||||||
|
res := CloneCompleteMultipartUploadOptions(opt)
|
||||||
|
if !reflect.DeepEqual(res, opt) {
|
||||||
|
t.Errorf("CloneCompleteMultipartUploadOptions, returned:%+v,want:%+v", res, opt)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(ori, *opt) {
|
||||||
|
t.Errorf("CloneCompleteMultipartUploadOptions, returned:%+v,want:%+v", *opt, ori)
|
||||||
|
}
|
||||||
|
res.XOptionHeader.Add("TestClone", "value")
|
||||||
|
if v := opt.XOptionHeader.Get("TestClone"); v != "" {
|
||||||
|
t.Errorf("CloneCompleteMultipartUploadOptions, returned:%+v,want:%+v", res, opt)
|
||||||
|
}
|
||||||
|
opt = &CompleteMultipartUploadOptions{}
|
||||||
|
res = CloneCompleteMultipartUploadOptions(opt)
|
||||||
|
if !reflect.DeepEqual(res, opt) {
|
||||||
|
t.Errorf("CloneCompleteMultipartUploadOptions, returned:%+v,want:%+v", res, opt)
|
||||||
|
}
|
||||||
|
res.Parts = append(res.Parts, Object{Key: "K", ETag: "T"})
|
||||||
|
if len(opt.Parts) > 0 {
|
||||||
|
t.Errorf("CloneCompleteMultipartUploadOptions Failed")
|
||||||
|
}
|
||||||
|
if reflect.DeepEqual(res, opt) {
|
||||||
|
t.Errorf("CloneCompleteMultipartUploadOptions, returned:%+v,want:%+v", res, opt)
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|||||||
@@ -402,37 +402,7 @@ func (s *ObjectService) innerHead(ctx context.Context, sourceURL string, opt *Ob
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
func SplitCopyFileIntoChunks(totalBytes int64, partSize int64) ([]Chunk, int, error) {
|
// 如果源对象大于5G,则采用分块复制的方式进行拷贝,此时源对象的元信息如果COPY
|
||||||
var partNum int64
|
|
||||||
if partSize > 0 {
|
|
||||||
partSize = partSize * 1024 * 1024
|
|
||||||
partNum = totalBytes / partSize
|
|
||||||
if partNum >= 10000 {
|
|
||||||
return nil, 0, errors.New("Too many parts, out of 10000")
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
partNum, partSize = DividePart(totalBytes, 128)
|
|
||||||
}
|
|
||||||
|
|
||||||
var chunks []Chunk
|
|
||||||
var chunk = Chunk{}
|
|
||||||
for i := int64(0); i < partNum; i++ {
|
|
||||||
chunk.Number = int(i + 1)
|
|
||||||
chunk.OffSet = i * partSize
|
|
||||||
chunk.Size = partSize
|
|
||||||
chunks = append(chunks, chunk)
|
|
||||||
}
|
|
||||||
|
|
||||||
if totalBytes%partSize > 0 {
|
|
||||||
chunk.Number = len(chunks) + 1
|
|
||||||
chunk.OffSet = int64(len(chunks)) * partSize
|
|
||||||
chunk.Size = totalBytes % partSize
|
|
||||||
chunks = append(chunks, chunk)
|
|
||||||
partNum++
|
|
||||||
}
|
|
||||||
return chunks, int(partNum), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *ObjectService) MultiCopy(ctx context.Context, name string, sourceURL string, opt *MultiCopyOptions, id ...string) (*ObjectCopyResult, *Response, error) {
|
func (s *ObjectService) MultiCopy(ctx context.Context, name string, sourceURL string, opt *MultiCopyOptions, id ...string) (*ObjectCopyResult, *Response, error) {
|
||||||
resp, err := s.innerHead(ctx, sourceURL, nil, id)
|
resp, err := s.innerHead(ctx, sourceURL, nil, id)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -455,7 +425,7 @@ func (s *ObjectService) MultiCopy(ctx context.Context, name string, sourceURL st
|
|||||||
if opt == nil {
|
if opt == nil {
|
||||||
opt = &MultiCopyOptions{}
|
opt = &MultiCopyOptions{}
|
||||||
}
|
}
|
||||||
chunks, partNum, err := SplitCopyFileIntoChunks(totalBytes, opt.PartSize)
|
chunks, partNum, err := SplitSizeIntoChunks(totalBytes, opt.PartSize*1024*1024)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user