tommy
/
cos-go-sdk-v5
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

add ces-kms

master
jojoliang 4 years ago
parent
6097da8969
commit
7134982d66
12 changed files with 2081 additions and 0 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 67
      crypto/aes_ctr.go
  2. 138
      crypto/aes_ctr_cipher.go
  3. 107
      crypto/aes_ctr_cipher_test.go
  4. 69
      crypto/cipher.go
  5. 228
      crypto/crypto_object.go
  6. 76
      crypto/crypto_object_part.go
  7. 387
      crypto/crypto_object_part_test.go
  8. 385
      crypto/crypto_object_test.go
  9. 141
      crypto/crypto_type.go
  10. 87
      crypto/master_kms_cipher.go
  11. 91
      crypto/master_kms_cipher_test.go
  12. 305
      example/crypto/crypto_sample.go

67
crypto/aes_ctr.go
View File

@ -0,0 +1,67 @@
package coscrypto
import (
"crypto/aes"
"crypto/cipher"
"io"
)
type aesCtr struct {
encrypter cipher.Stream
decrypter cipher.Stream
}
func newAesCtr(cd CipherData) (Cipher, error) {
block, err := aes.NewCipher(cd.Key)
if err != nil {
return nil, err
}
encrypter := cipher.NewCTR(block, cd.IV)
decrypter := cipher.NewCTR(block, cd.IV)
return &aesCtr{encrypter, decrypter}, nil
}
func (c *aesCtr) Encrypt(src io.Reader) io.Reader {
reader := &ctrEncryptReader{
encrypter: c.encrypter,
src: src,
}
return reader
}
type ctrEncryptReader struct {
encrypter cipher.Stream
src io.Reader
}
func (reader *ctrEncryptReader) Read(data []byte) (int, error) {
plainText := make([]byte, len(data), len(data))
n, err := reader.src.Read(plainText)
if n > 0 {
plainText = plainText[0:n]
reader.encrypter.XORKeyStream(data, plainText)
}
return n, err
}
func (c *aesCtr) Decrypt(src io.Reader) io.Reader {
return &ctrDecryptReader{
decrypter: c.decrypter,
src: src,
}
}
type ctrDecryptReader struct {
decrypter cipher.Stream
src io.Reader
}
func (reader *ctrDecryptReader) Read(data []byte) (int, error) {
cryptoText := make([]byte, len(data), len(data))
n, err := reader.src.Read(cryptoText)
if n > 0 {
cryptoText = cryptoText[0:n]
reader.decrypter.XORKeyStream(data, cryptoText)
}
return n, err
}

138
crypto/aes_ctr_cipher.go
View File

@ -0,0 +1,138 @@
package coscrypto
import (
"io"
)
const (
aesKeySize = 32
ivSize = 16
)
type aesCtrCipherBuilder struct {
MasterCipher MasterCipher
}
type aesCtrCipher struct {
CipherData CipherData
Cipher Cipher
}
func CreateAesCtrBuilder(cipher MasterCipher) ContentCipherBuilder {
return aesCtrCipherBuilder{MasterCipher: cipher}
}
func (builder aesCtrCipherBuilder) createCipherData() (CipherData, error) {
var cd CipherData
var err error
err = cd.RandomKeyIv(aesKeySize, ivSize)
if err != nil {
return cd, err
}
cd.WrapAlgorithm = builder.MasterCipher.GetWrapAlgorithm()
cd.CEKAlgorithm = AesCtrAlgorithm
cd.MatDesc = builder.MasterCipher.GetMatDesc()
// EncryptedKey
cd.EncryptedKey, err = builder.MasterCipher.Encrypt(cd.Key)
if err != nil {
return cd, err
}
// EncryptedIV
cd.EncryptedIV, err = builder.MasterCipher.Encrypt(cd.IV)
if err != nil {
return cd, err
}
return cd, nil
}
func (builder aesCtrCipherBuilder) contentCipherCD(cd CipherData) (ContentCipher, error) {
cipher, err := newAesCtr(cd)
if err != nil {
return nil, err
}
return &aesCtrCipher{
CipherData: cd,
Cipher: cipher,
}, nil
}
func (builder aesCtrCipherBuilder) ContentCipher() (ContentCipher, error) {
cd, err := builder.createCipherData()
if err != nil {
return nil, err
}
return builder.contentCipherCD(cd)
}
func (builder aesCtrCipherBuilder) ContentCipherEnv(envelope Envelope) (ContentCipher, error) {
var cd CipherData
cd.EncryptedKey = make([]byte, len(envelope.CipherKey))
copy(cd.EncryptedKey, []byte(envelope.CipherKey))
plainKey, err := builder.MasterCipher.Decrypt([]byte(envelope.CipherKey))
if err != nil {
return nil, err
}
cd.Key = make([]byte, len(plainKey))
copy(cd.Key, plainKey)
cd.EncryptedIV = make([]byte, len(envelope.IV))
copy(cd.EncryptedIV, []byte(envelope.IV))
plainIV, err := builder.MasterCipher.Decrypt([]byte(envelope.IV))
if err != nil {
return nil, err
}
cd.IV = make([]byte, len(plainIV))
copy(cd.IV, plainIV)
cd.MatDesc = envelope.MatDesc
cd.WrapAlgorithm = envelope.WrapAlg
cd.CEKAlgorithm = envelope.CEKAlg
return builder.contentCipherCD(cd)
}
func (builder aesCtrCipherBuilder) GetMatDesc() string {
return builder.MasterCipher.GetMatDesc()
}
func (cc *aesCtrCipher) EncryptContent(src io.Reader) (io.ReadCloser, error) {
reader := cc.Cipher.Encrypt(src)
return &CryptoEncrypter{Body: src, Encrypter: reader}, nil
}
func (cc *aesCtrCipher) DecryptContent(src io.Reader) (io.ReadCloser, error) {
reader := cc.Cipher.Decrypt(src)
return &CryptoDecrypter{Body: src, Decrypter: reader}, nil
}
func (cc *aesCtrCipher) GetCipherData() *CipherData {
return &(cc.CipherData)
}
func (cc *aesCtrCipher) GetEncryptedLen(plainTextLen int64) int64 {
return plainTextLen
}
func (cc *aesCtrCipher) GetAlignLen() int {
return len(cc.CipherData.IV)
}
func (cc *aesCtrCipher) Clone(cd CipherData) (ContentCipher, error) {
cipher, err := newAesCtr(cd)
if err != nil {
return nil, err
}
return &aesCtrCipher{
CipherData: cd,
Cipher: cipher,
}, nil
}

107
crypto/aes_ctr_cipher_test.go
View File

@ -0,0 +1,107 @@
package coscrypto_test
import (
"bytes"
"crypto/aes"
"crypto/cipher"
"crypto/rand"
"github.com/stretchr/testify/assert"
"github.com/tencentyun/cos-go-sdk-v5/crypto"
"io/ioutil"
math_rand "math/rand"
)
type EmptyMasterCipher struct{}
func (mc EmptyMasterCipher) Encrypt(b []byte) ([]byte, error) {
return b, nil
}
func (mc EmptyMasterCipher) Decrypt(b []byte) ([]byte, error) {
return b, nil
}
func (mc EmptyMasterCipher) GetWrapAlgorithm() string {
return "Test/EmptyWrapAlgo"
}
func (mc EmptyMasterCipher) GetMatDesc() string {
return "Empty Desc"
}
func (s *CosTestSuite) TestCryptoObjectService_EncryptAndDecrypt() {
var masterCipher EmptyMasterCipher
builder := coscrypto.CreateAesCtrBuilder(masterCipher)
contentCipher, err := builder.ContentCipher()
assert.Nil(s.T(), err, "CryptoObject.CreateAesCtrBuilder Failed")
dataSize := math_rand.Int63n(1024 * 1024 * 32)
originData := make([]byte, dataSize)
rand.Read(originData)
// 加密
r1 := bytes.NewReader(originData)
reader1, err := contentCipher.EncryptContent(r1)
assert.Nil(s.T(), err, "CryptoObject.contentCipher.Encrypt Failed")
encryptedData, err := ioutil.ReadAll(reader1)
assert.Nil(s.T(), err, "CryptoObject.Read Failed")
// 解密
r2 := bytes.NewReader(encryptedData)
reader2, err := contentCipher.DecryptContent(r2)
decryptedData, err := ioutil.ReadAll(reader2)
assert.Nil(s.T(), err, "CryptoObject.Read Failed")
assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData")
}
func (s *CosTestSuite) TestCryptoObjectService_Encrypt() {
var masterCipher EmptyMasterCipher
builder := coscrypto.CreateAesCtrBuilder(masterCipher)
contentCipher, err := builder.ContentCipher()
assert.Nil(s.T(), err, "CryptoObject.CreateAesCtrBuilder Failed")
dataSize := math_rand.Int63n(1024 * 1024 * 32)
originData := make([]byte, dataSize)
rand.Read(originData)
// 加密
r := bytes.NewReader(originData)
reader, err := contentCipher.EncryptContent(r)
assert.Nil(s.T(), err, "CryptoObject.contentCipher.Encrypt Failed")
encryptedData, err := ioutil.ReadAll(reader)
assert.Nil(s.T(), err, "CryptoObject.Read Failed")
// 直接解密
cd := contentCipher.GetCipherData()
block, err := aes.NewCipher(cd.Key)
assert.Nil(s.T(), err, "CryptoObject.NewCipher Failed")
decrypter := cipher.NewCTR(block, cd.IV)
decryptedData := make([]byte, len(originData))
decrypter.XORKeyStream(decryptedData, encryptedData)
assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData")
}
func (s *CosTestSuite) TestCryptoObjectService_Decrypt() {
var masterCipher EmptyMasterCipher
builder := coscrypto.CreateAesCtrBuilder(masterCipher)
contentCipher, err := builder.ContentCipher()
assert.Nil(s.T(), err, "CryptoObject.CreateAesCtrBuilder Failed")
dataSize := math_rand.Int63n(1024 * 1024 * 32)
originData := make([]byte, dataSize)
rand.Read(originData)
// 直接加密
cd := contentCipher.GetCipherData()
block, err := aes.NewCipher(cd.Key)
assert.Nil(s.T(), err, "CryptoObject.NewCipher Failed")
encrypter := cipher.NewCTR(block, cd.IV)
encryptedData := make([]byte, len(originData))
encrypter.XORKeyStream(encryptedData, originData)
// 解密
r := bytes.NewReader(encryptedData)
reader, err := contentCipher.DecryptContent(r)
assert.Nil(s.T(), err, "CryptoObject.contentCipher.Encrypt Failed")
decryptedData, err := ioutil.ReadAll(reader)
assert.Nil(s.T(), err, "CryptoObject.Read Failed")
assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData")
}

69
crypto/cipher.go
View File

@ -0,0 +1,69 @@
package coscrypto
import (
"io"
)
// Cipher is interface for encryption or decryption of an object
type Cipher interface {
Encrypter
Decrypter
}
// Encrypter is interface with only encrypt method
type Encrypter interface {
Encrypt(io.Reader) io.Reader
}
// Decrypter is interface with only decrypt method
type Decrypter interface {
Decrypt(io.Reader) io.Reader
}
// CryptoEncrypter provides close method for Encrypter
type CryptoEncrypter struct {
Body io.Reader
Encrypter io.Reader
isClosed bool
}
// Close lets the CryptoEncrypter satisfy io.ReadCloser interface
func (rc *CryptoEncrypter) Close() error {
rc.isClosed = true
if closer, ok := rc.Body.(io.ReadCloser); ok {
return closer.Close()
}
return nil
}
// Read lets the CryptoEncrypter satisfy io.ReadCloser interface
func (rc *CryptoEncrypter) Read(b []byte) (int, error) {
if rc.isClosed {
return 0, io.EOF
}
return rc.Encrypter.Read(b)
}
// CryptoDecrypter provides close method for Decrypter
type CryptoDecrypter struct {
Body io.Reader
Decrypter io.Reader
isClosed bool
}
// Close lets the CryptoDecrypter satisfy io.ReadCloser interface
func (rc *CryptoDecrypter) Close() error {
rc.isClosed = true
if closer, ok := rc.Body.(io.ReadCloser); ok {
return closer.Close()
}
return nil
}
// Read lets the CryptoDecrypter satisfy io.ReadCloser interface
func (rc *CryptoDecrypter) Read(b []byte) (int, error) {
if rc.isClosed {
return 0, io.EOF
}
return rc.Decrypter.Read(b)
}

228
crypto/crypto_object.go
View File

@ -0,0 +1,228 @@
package coscrypto
import (
"context"
"fmt"
"github.com/tencentyun/cos-go-sdk-v5"
"io"
"net/http"
"os"
"strconv"
)
type CryptoObjectService struct {
*cos.ObjectService
cryptoClient *CryptoClient
}
type CryptoClient struct {
*cos.Client
Object *CryptoObjectService
ContentCipherBuilder ContentCipherBuilder
userAgent string
}
func NewCryptoClient(client *cos.Client, masterCipher MasterCipher) *CryptoClient {
cc := &CryptoClient{
Client: client,
Object: &CryptoObjectService{
client.Object,
nil,
},
ContentCipherBuilder: CreateAesCtrBuilder(masterCipher),
}
cc.userAgent = cc.Client.UserAgent + "/" + EncryptionUaSuffix
cc.Object.cryptoClient = cc
return cc
}
func (s *CryptoObjectService) Put(ctx context.Context, name string, r io.Reader, opt *cos.ObjectPutOptions) (*cos.Response, error) {
cc, err := s.cryptoClient.ContentCipherBuilder.ContentCipher()
if err != nil {
return nil, err
}
reader, err := cc.EncryptContent(r)
if err != nil {
return nil, err
}
opt = cos.CloneObjectPutOptions(opt)
totalBytes, err := cos.GetReaderLen(r)
if err != nil && opt != nil && opt.Listener != nil && opt.ContentLength == 0 {
return nil, err
}
if err == nil {
if opt != nil && opt.ContentLength == 0 {
// 如果未设置Listener, 非bytes.Buffer/bytes.Reader/strings.Reader由用户指定Contength
if opt.Listener != nil || cos.IsLenReader(r) {
opt.ContentLength = totalBytes
}
}
}
if opt.XOptionHeader == nil {
opt.XOptionHeader = &http.Header{}
}
if opt.ContentMD5 != "" {
opt.XOptionHeader.Add(COSClientSideEncryptionUnencryptedContentMD5, opt.ContentMD5)
opt.ContentMD5 = ""
}
if opt.ContentLength != 0 {
opt.XOptionHeader.Add(COSClientSideEncryptionUnencryptedContentLength, strconv.FormatInt(opt.ContentLength, 10))
opt.ContentLength = cc.GetEncryptedLen(opt.ContentLength)
}
addCryptoHeaders(opt.XOptionHeader, cc.GetCipherData())
return s.ObjectService.Put(ctx, name, reader, opt)
}
func (s *CryptoObjectService) PutFromFile(ctx context.Context, name, filePath string, opt *cos.ObjectPutOptions) (resp *cos.Response, err error) {
nr := 0
for nr < 3 {
fd, e := os.Open(filePath)
if e != nil {
err = e
return
}
resp, err = s.Put(ctx, name, fd, opt)
if err != nil {
nr++
fd.Close()
continue
}
fd.Close()
break
}
return
}
func (s *CryptoObjectService) Get(ctx context.Context, name string, opt *cos.ObjectGetOptions) (*cos.Response, error) {
meta, err := s.ObjectService.Head(ctx, name, nil)
if err != nil {
return meta, err
}
_isEncrypted := isEncrypted(&meta.Header)
if !_isEncrypted {
return s.ObjectService.Get(ctx, name, opt)
}
envelope := getEnvelopeFromHeader(&meta.Header)
if !envelope.IsValid() {
return nil, fmt.Errorf("get envelope from header failed, object:%v", name)
}
encryptMatDesc := s.cryptoClient.ContentCipherBuilder.GetMatDesc()
if envelope.MatDesc != encryptMatDesc {
return nil, fmt.Errorf("provided master cipher error, want:%v, return:%v, object:%v", encryptMatDesc, envelope.MatDesc, name)
}
cc, err := s.cryptoClient.ContentCipherBuilder.ContentCipherEnv(envelope)
if err != nil {
return nil, fmt.Errorf("get content cipher from envelope failed: %v, object:%v", err, name)
}
optRange, err := cos.GetRangeOptions(opt)
if err != nil {
return nil, err
}
discardAlignLen := int64(0)
// Range请求
if optRange != nil && optRange.HasStart {
// 加密block对齐
adjustStart := adjustRangeStart(optRange.Start, int64(cc.GetAlignLen()))
discardAlignLen = optRange.Start - adjustStart
if discardAlignLen > 0 {
optRange.Start = adjustStart
opt = cos.CloneObjectGetOptions(opt)
opt.Range = cos.FormatRangeOptions(optRange)
}
cd := cc.GetCipherData().Clone()
cd.SeekIV(uint64(adjustStart))
cc, err = cc.Clone(cd)
if err != nil {
return nil, fmt.Errorf("ContentCipher Clone failed:%v, bject:%v", err, name)
}
}
resp, err := s.ObjectService.Get(ctx, name, opt)
if err != nil {
return resp, err
}
resp.Body, err = cc.DecryptContent(resp.Body)
if err != nil {
return resp, err
}
// 抛弃多读取的数据
if discardAlignLen > 0 {
resp.Body = &cos.DiscardReadCloser{
RC: resp.Body,
Discard: int(discardAlignLen),
}
}
return resp, err
}
func (s *CryptoObjectService) GetToFile(ctx context.Context, name, localpath string, opt *cos.ObjectGetOptions) (*cos.Response, error) {
resp, err := s.Get(ctx, name, opt)
if err != nil {
return resp, err
}
defer resp.Body.Close()
// If file exist, overwrite it
fd, err := os.OpenFile(localpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660)
if err != nil {
return resp, err
}
_, err = io.Copy(fd, resp.Body)
fd.Close()
if err != nil {
return resp, err
}
return resp, nil
}
func (s *CryptoObjectService) MultiUpload(ctx context.Context, name string, filepath string, opt *cos.MultiUploadOptions) (*cos.CompleteMultipartUploadResult, *cos.Response, error) {
return s.Upload(ctx, name, filepath, opt)
}
func (s *CryptoObjectService) Upload(ctx context.Context, name string, filepath string, opt *cos.MultiUploadOptions) (*cos.CompleteMultipartUploadResult, *cos.Response, error) {
return nil, nil, fmt.Errorf("CryptoObjectService doesn't support Upload Now")
}
func (s *CryptoObjectService) Download(ctx context.Context, name string, filepath string, opt *cos.MultiDownloadOptions) (*cos.Response, error) {
return nil, fmt.Errorf("CryptoObjectService doesn't support Download Now")
}
func adjustRangeStart(start int64, alignLen int64) int64 {
return (start / alignLen) * alignLen
}
func addCryptoHeaders(header *http.Header, cd *CipherData) {
if cd.MatDesc != "" {
header.Add(COSClientSideEncryptionMatDesc, cd.MatDesc)
}
header.Add(COSClientSideEncryptionKey, string(cd.EncryptedKey))
header.Add(COSClientSideEncryptionStart, string(cd.EncryptedIV))
header.Add(COSClientSideEncryptionWrapAlg, cd.WrapAlgorithm)
header.Add(COSClientSideEncryptionCekAlg, cd.CEKAlgorithm)
}
func getEnvelopeFromHeader(header *http.Header) Envelope {
var envelope Envelope
envelope.CipherKey = header.Get(COSClientSideEncryptionKey)
envelope.IV = header.Get(COSClientSideEncryptionStart)
envelope.MatDesc = header.Get(COSClientSideEncryptionMatDesc)
envelope.WrapAlg = header.Get(COSClientSideEncryptionWrapAlg)
envelope.CEKAlg = header.Get(COSClientSideEncryptionCekAlg)
return envelope
}
func isEncrypted(header *http.Header) bool {
encryptedKey := header.Get(COSClientSideEncryptionKey)
if len(encryptedKey) > 0 {
return true
}
return false
}

76
crypto/crypto_object_part.go
View File

@ -0,0 +1,76 @@
package coscrypto
import (
"context"
"fmt"
"github.com/tencentyun/cos-go-sdk-v5"
"io"
"net/http"
"strconv"
)
type CryptoContext struct {
DataSize int64
PartSize int64
ContentCipher ContentCipher
}
func partSizeIsValid(partSize int64, alignLen int64) bool {
if partSize%alignLen == 0 {
return true
}
return false
}
func (s *CryptoObjectService) InitiateMultipartUpload(ctx context.Context, name string, opt *cos.InitiateMultipartUploadOptions, cryptoCtx *CryptoContext) (*cos.InitiateMultipartUploadResult, *cos.Response, error) {
contentCipher, err := s.cryptoClient.ContentCipherBuilder.ContentCipher()
if err != nil {
return nil, nil, err
}
if !partSizeIsValid(cryptoCtx.PartSize, int64(contentCipher.GetAlignLen())) {
return nil, nil, fmt.Errorf("PartSize is invalid, it should be %v aligned", contentCipher.GetAlignLen())
}
// 添加自定义头部
cryptoCtx.ContentCipher = contentCipher
opt = cos.CloneInitiateMultipartUploadOptions(opt)
if opt.XOptionHeader == nil {
opt.XOptionHeader = &http.Header{}
}
if opt.ContentMD5 != "" {
opt.XOptionHeader.Add(COSClientSideEncryptionUnencryptedContentMD5, opt.ContentMD5)
opt.ContentMD5 = ""
}
opt.XOptionHeader.Add(COSClientSideEncryptionUnencryptedContentLength, strconv.FormatInt(cryptoCtx.DataSize, 10))
addCryptoHeaders(opt.XOptionHeader, contentCipher.GetCipherData())
return s.ObjectService.InitiateMultipartUpload(ctx, name, opt)
}
func (s *CryptoObjectService) UploadPart(ctx context.Context, name, uploadID string, partNumber int, r io.Reader, opt *cos.ObjectUploadPartOptions, cryptoCtx *CryptoContext) (*cos.Response, error) {
if cryptoCtx.PartSize == 0 {
return nil, fmt.Errorf("CryptoContext's PartSize is zero")
}
opt = cos.CloneObjectUploadPartOptions(opt)
if cryptoCtx.ContentCipher == nil {
return nil, fmt.Errorf("ContentCipher is nil, Please call the InitiateMultipartUpload")
}
totalBytes, err := cos.GetReaderLen(r)
if err == nil {
// 与 go http 保持一致, 非bytes.Buffer/bytes.Reader/strings.Reader需用户指定ContentLength
if opt != nil && opt.ContentLength == 0 && cos.IsLenReader(r) {
opt.ContentLength = totalBytes
}
}
cd := cryptoCtx.ContentCipher.GetCipherData().Clone()
cd.SeekIV(uint64(partNumber-1) * uint64(cryptoCtx.PartSize))
cc, err := cryptoCtx.ContentCipher.Clone(cd)
opt.ContentLength = cc.GetEncryptedLen(opt.ContentLength)
if err != nil {
return nil, err
}
reader, err := cc.EncryptContent(r)
if err != nil {
return nil, err
}
return s.ObjectService.UploadPart(ctx, name, uploadID, partNumber, reader, opt)
}

387
crypto/crypto_object_part_test.go
View File

@ -0,0 +1,387 @@
package coscrypto_test
import (
"bytes"
"context"
"crypto/aes"
"crypto/cipher"
"crypto/md5"
"crypto/rand"
"encoding/base64"
"fmt"
"github.com/stretchr/testify/assert"
"github.com/tencentyun/cos-go-sdk-v5"
"github.com/tencentyun/cos-go-sdk-v5/crypto"
"io"
"io/ioutil"
math_rand "math/rand"
"net/http"
"net/url"
"os"
"sort"
"sync"
"time"
)
func (s *CosTestSuite) TestMultiUpload_Normal() {
name := "test/ObjectPut" + time.Now().Format(time.RFC3339)
contentLength := int64(1024*1024*10 + 1)
originData := make([]byte, contentLength)
_, err := rand.Read(originData)
cryptoCtx := coscrypto.CryptoContext{
DataSize: contentLength,
PartSize: (contentLength / 16 / 3) * 16,
}
v, _, err := s.CClient.Object.InitiateMultipartUpload(context.Background(), name, nil, &cryptoCtx)
assert.Nil(s.T(), err, "Init Failed")
chunks, _, err := cos.SplitSizeIntoChunks(contentLength, cryptoCtx.PartSize)
assert.Nil(s.T(), err, "Split Failed")
optcom := &cos.CompleteMultipartUploadOptions{}
for _, chunk := range chunks {
opt := &cos.ObjectUploadPartOptions{
ContentLength: chunk.Size,
}
f := bytes.NewReader(originData[chunk.OffSet : chunk.OffSet+chunk.Size])
resp, err := s.CClient.Object.UploadPart(context.Background(), name, v.UploadID, chunk.Number, io.LimitReader(f, chunk.Size), opt, &cryptoCtx)
assert.Nil(s.T(), err, "UploadPart failed")
optcom.Parts = append(optcom.Parts, cos.Object{
PartNumber: chunk.Number, ETag: resp.Header.Get("ETag"),
})
}
_, _, err = s.CClient.Object.CompleteMultipartUpload(context.Background(), name, v.UploadID, optcom)
assert.Nil(s.T(), err, "Complete Failed")
resp, err := s.CClient.Object.Get(context.Background(), name, nil)
assert.Nil(s.T(), err, "GetObject Failed")
defer resp.Body.Close()
decryptedData, _ := ioutil.ReadAll(resp.Body)
assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData")
_, err = s.CClient.Object.Delete(context.Background(), name)
assert.Nil(s.T(), err, "DeleteObject Failed")
}
func (s *CosTestSuite) TestMultiUpload_DecryptWithKey() {
name := "test/ObjectPut" + time.Now().Format(time.RFC3339)
contentLength := int64(1024*1024*10 + 1)
originData := make([]byte, contentLength)
_, err := rand.Read(originData)
f := bytes.NewReader(originData)
// 分块上传
cryptoCtx := coscrypto.CryptoContext{
DataSize: contentLength,
PartSize: (contentLength / 16 / 3) * 16,
}
v, _, err := s.CClient.Object.InitiateMultipartUpload(context.Background(), name, nil, &cryptoCtx)
assert.Nil(s.T(), err, "Init Failed")
chunks, _, err := cos.SplitSizeIntoChunks(contentLength, cryptoCtx.PartSize)
assert.Nil(s.T(), err, "Split Failed")
optcom := &cos.CompleteMultipartUploadOptions{}
for _, chunk := range chunks {
opt := &cos.ObjectUploadPartOptions{
ContentLength: chunk.Size,
Listener: &cos.DefaultProgressListener{},
}
resp, err := s.CClient.Object.UploadPart(context.Background(), name, v.UploadID, chunk.Number, io.LimitReader(f, chunk.Size), opt, &cryptoCtx)
assert.Nil(s.T(), err, "UploadPart failed")
optcom.Parts = append(optcom.Parts, cos.Object{
PartNumber: chunk.Number, ETag: resp.Header.Get("ETag"),
})
}
_, _, err = s.CClient.Object.CompleteMultipartUpload(context.Background(), name, v.UploadID, optcom)
assert.Nil(s.T(), err, "Complete Failed")
// 正常读取
resp, err := s.Client.Object.Get(context.Background(), name, nil)
assert.Nil(s.T(), err, "GetObject Failed")
defer resp.Body.Close()
encryptedData, _ := ioutil.ReadAll(resp.Body)
assert.NotEqual(s.T(), bytes.Compare(encryptedData, originData), 0, "encryptedData == originData")
// 获取解密信息
resp, err = s.CClient.Object.Head(context.Background(), name, nil)
assert.Nil(s.T(), err, "HeadObject Failed")
cipherKey := resp.Header.Get(coscrypto.COSClientSideEncryptionKey)
cipherIV := resp.Header.Get(coscrypto.COSClientSideEncryptionStart)
key, err := s.Master.Decrypt([]byte(cipherKey))
assert.Nil(s.T(), err, "Master Decrypt Failed")
iv, err := s.Master.Decrypt([]byte(cipherIV))
assert.Nil(s.T(), err, "Master Decrypt Failed")
// 手动解密
block, err := aes.NewCipher(key)
assert.Nil(s.T(), err, "NewCipher Failed")
decrypter := cipher.NewCTR(block, iv)
decryptedData := make([]byte, len(originData))
decrypter.XORKeyStream(decryptedData, encryptedData)
assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData")
_, err = s.CClient.Object.Delete(context.Background(), name)
assert.Nil(s.T(), err, "DeleteObject Failed")
}
func (s *CosTestSuite) TestMultiUpload_PutFromFile() {
name := "test/ObjectPut" + time.Now().Format(time.RFC3339)
filepath := "tmpfile" + time.Now().Format(time.RFC3339)
newfile, err := os.Create(filepath)
assert.Nil(s.T(), err, "Create File Failed")
defer os.Remove(filepath)
contentLength := int64(1024*1024*10 + 1)
originData := make([]byte, contentLength)
_, err = rand.Read(originData)
newfile.Write(originData)
newfile.Close()
m := md5.New()
m.Write(originData)
contentMD5 := m.Sum(nil)
cryptoCtx := coscrypto.CryptoContext{
DataSize: contentLength,
PartSize: (contentLength / 16 / 3) * 16,
}
v, _, err := s.CClient.Object.InitiateMultipartUpload(context.Background(), name, nil, &cryptoCtx)
assert.Nil(s.T(), err, "Init Failed")
_, chunks, _, err := cos.SplitFileIntoChunks(filepath, cryptoCtx.PartSize)
assert.Nil(s.T(), err, "Split Failed")
optcom := &cos.CompleteMultipartUploadOptions{}
var wg sync.WaitGroup
var mtx sync.Mutex
for _, chunk := range chunks {
wg.Add(1)
go func(chk cos.Chunk) {
defer wg.Done()
fd, err := os.Open(filepath)
assert.Nil(s.T(), err, "Open File Failed")
opt := &cos.ObjectUploadPartOptions{
ContentLength: chk.Size,
}
fd.Seek(chk.OffSet, os.SEEK_SET)
resp, err := s.CClient.Object.UploadPart(context.Background(), name, v.UploadID, chk.Number, io.LimitReader(fd, chk.Size), opt, &cryptoCtx)
assert.Nil(s.T(), err, "UploadPart failed")
mtx.Lock()
optcom.Parts = append(optcom.Parts, cos.Object{
PartNumber: chk.Number, ETag: resp.Header.Get("ETag"),
})
mtx.Unlock()
}(chunk)
}
wg.Wait()
sort.Sort(cos.ObjectList(optcom.Parts))
_, _, err = s.CClient.Object.CompleteMultipartUpload(context.Background(), name, v.UploadID, optcom)
assert.Nil(s.T(), err, "Complete Failed")
downfile := "downfile" + time.Now().Format(time.RFC3339)
_, err = s.CClient.Object.GetToFile(context.Background(), name, downfile, nil)
assert.Nil(s.T(), err, "GetObject Failed")
m = md5.New()
fd, err := os.Open(downfile)
assert.Nil(s.T(), err, "Open File Failed")
defer os.Remove(downfile)
defer fd.Close()
io.Copy(m, fd)
downContentMD5 := m.Sum(nil)
assert.Equal(s.T(), bytes.Compare(contentMD5, downContentMD5), 0, "decryptData != originData")
_, err = s.CClient.Object.Delete(context.Background(), name)
assert.Nil(s.T(), err, "DeleteObject Failed")
}
func (s *CosTestSuite) TestMultiUpload_GetWithRange() {
name := "test/ObjectPut" + time.Now().Format(time.RFC3339)
filepath := "tmpfile" + time.Now().Format(time.RFC3339)
newfile, err := os.Create(filepath)
assert.Nil(s.T(), err, "Create File Failed")
defer os.Remove(filepath)
contentLength := int64(1024*1024*10 + 1)
originData := make([]byte, contentLength)
_, err = rand.Read(originData)
newfile.Write(originData)
newfile.Close()
m := md5.New()
m.Write(originData)
contentMD5 := m.Sum(nil)
cryptoCtx := coscrypto.CryptoContext{
DataSize: contentLength,
PartSize: (contentLength / 16 / 3) * 16,
}
iniopt := &cos.InitiateMultipartUploadOptions{
&cos.ACLHeaderOptions{
XCosACL: "private",
},
&cos.ObjectPutHeaderOptions{
ContentMD5: base64.StdEncoding.EncodeToString(contentMD5),
XCosMetaXXX: &http.Header{},
},
}
iniopt.XCosMetaXXX.Add("x-cos-meta-isEncrypted", "true")
v, _, err := s.CClient.Object.InitiateMultipartUpload(context.Background(), name, iniopt, &cryptoCtx)
assert.Nil(s.T(), err, "Init Failed")
_, chunks, _, err := cos.SplitFileIntoChunks(filepath, cryptoCtx.PartSize)
assert.Nil(s.T(), err, "Split Failed")
optcom := &cos.CompleteMultipartUploadOptions{}
var wg sync.WaitGroup
var mtx sync.Mutex
for _, chunk := range chunks {
wg.Add(1)
go func(chk cos.Chunk) {
defer wg.Done()
fd, err := os.Open(filepath)
assert.Nil(s.T(), err, "Open File Failed")
opt := &cos.ObjectUploadPartOptions{
ContentLength: chk.Size,
}
fd.Seek(chk.OffSet, os.SEEK_SET)
resp, err := s.CClient.Object.UploadPart(context.Background(), name, v.UploadID, chk.Number, io.LimitReader(fd, chk.Size), opt, &cryptoCtx)
assert.Nil(s.T(), err, "UploadPart failed")
mtx.Lock()
optcom.Parts = append(optcom.Parts, cos.Object{
PartNumber: chk.Number, ETag: resp.Header.Get("ETag"),
})
mtx.Unlock()
}(chunk)
}
wg.Wait()
sort.Sort(cos.ObjectList(optcom.Parts))
_, _, err = s.CClient.Object.CompleteMultipartUpload(context.Background(), name, v.UploadID, optcom)
assert.Nil(s.T(), err, "Complete Failed")
// Range解密读取
for i := 0; i < 10; i++ {
math_rand.Seed(time.Now().UnixNano())
rangeStart := math_rand.Int63n(contentLength)
rangeEnd := rangeStart + math_rand.Int63n(contentLength-rangeStart)
if rangeEnd == rangeStart || rangeStart >= contentLength-1 {
continue
}
opt := &cos.ObjectGetOptions{
Range: fmt.Sprintf("bytes=%v-%v", rangeStart, rangeEnd),
Listener: &cos.DefaultProgressListener{},
}
resp, err := s.CClient.Object.Get(context.Background(), name, opt)
assert.Nil(s.T(), err, "GetObject Failed")
defer resp.Body.Close()
decryptedData, _ := ioutil.ReadAll(resp.Body)
assert.Equal(s.T(), bytes.Compare(originData[rangeStart:rangeEnd+1], decryptedData), 0, "decryptData != originData")
}
opt := &cos.ObjectGetOptions{
Listener: &cos.DefaultProgressListener{},
}
resp, err := s.CClient.Object.Get(context.Background(), name, opt)
assert.Nil(s.T(), err, "GetObject Failed")
assert.Equal(s.T(), resp.Header.Get("x-cos-meta-isEncrypted"), "true", "meta data isn't consistent")
assert.Equal(s.T(), resp.Header.Get(coscrypto.COSClientSideEncryptionCekAlg), "AES/CTR/NoPadding", "meta data isn't consistent")
assert.Equal(s.T(), resp.Header.Get(coscrypto.COSClientSideEncryptionWrapAlg), "COS/KMS/Crypto", "meta data isn't consistent")
assert.Equal(s.T(), resp.Header.Get(coscrypto.COSClientSideEncryptionUnencryptedContentMD5), base64.StdEncoding.EncodeToString(contentMD5), "meta data isn't consistent")
defer resp.Body.Close()
decryptedData, _ := ioutil.ReadAll(resp.Body)
assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData")
_, err = s.CClient.Object.Delete(context.Background(), name)
assert.Nil(s.T(), err, "DeleteObject Failed")
}
func (s *CosTestSuite) TestMultiUpload_GetWithNewClient() {
name := "test/ObjectPut" + time.Now().Format(time.RFC3339)
contentLength := int64(1024*1024*10 + 1)
originData := make([]byte, contentLength)
_, err := rand.Read(originData)
cryptoCtx := coscrypto.CryptoContext{
DataSize: contentLength,
PartSize: (contentLength / 16 / 3) * 16,
}
v, _, err := s.CClient.Object.InitiateMultipartUpload(context.Background(), name, nil, &cryptoCtx)
assert.Nil(s.T(), err, "Init Failed")
chunks, _, err := cos.SplitSizeIntoChunks(contentLength, cryptoCtx.PartSize)
assert.Nil(s.T(), err, "Split Failed")
optcom := &cos.CompleteMultipartUploadOptions{}
for _, chunk := range chunks {
opt := &cos.ObjectUploadPartOptions{
ContentLength: chunk.Size,
}
f := bytes.NewReader(originData[chunk.OffSet : chunk.OffSet+chunk.Size])
resp, err := s.CClient.Object.UploadPart(context.Background(), name, v.UploadID, chunk.Number, io.LimitReader(f, chunk.Size), opt, &cryptoCtx)
assert.Nil(s.T(), err, "UploadPart failed")
optcom.Parts = append(optcom.Parts, cos.Object{
PartNumber: chunk.Number, ETag: resp.Header.Get("ETag"),
})
}
_, _, err = s.CClient.Object.CompleteMultipartUpload(context.Background(), name, v.UploadID, optcom)
assert.Nil(s.T(), err, "Complete Failed")
u, _ := url.Parse("https://" + kBucket + ".cos." + kRegion + ".myqcloud.com")
b := &cos.BaseURL{BucketURL: u}
c := cos.NewClient(b, &http.Client{
Transport: &cos.AuthorizationTransport{
SecretID: os.Getenv("COS_SECRETID"),
SecretKey: os.Getenv("COS_SECRETKEY"),
},
})
{
// 使用不同的MatDesc客户端读取, 期待错误
material := make(map[string]string)
material["desc"] = "cos crypto suite test 2"
kmsclient, _ := coscrypto.NewKMSClient(c.GetCredential(), kRegion)
master, _ := coscrypto.CreateMasterKMS(kmsclient, os.Getenv("KMSID"), material)
client := coscrypto.NewCryptoClient(c, master)
resp, err := client.Object.Get(context.Background(), name, nil)
assert.Nil(s.T(), resp, "Get Object Failed")
assert.NotNil(s.T(), err, "Get Object Failed")
}
{
// 使用相同的MatDesc客户端读取, 但KMSID不一样，期待正确，kms解密是不需要KMSID
material := make(map[string]string)
material["desc"] = "cos crypto suite test"
kmsclient, _ := coscrypto.NewKMSClient(c.GetCredential(), kRegion)
master, _ := coscrypto.CreateMasterKMS(kmsclient, "KMSID", material)
client := coscrypto.NewCryptoClient(c, master)
resp, err := client.Object.Get(context.Background(), name, nil)
assert.Nil(s.T(), err, "Get Object Failed")
defer resp.Body.Close()
decryptedData, _ := ioutil.ReadAll(resp.Body)
assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData")
}
{
// 使用相同的MatDesc客户端读取, 地域不一样，期待错误
material := make(map[string]string)
material["desc"] = "cos crypto suite test"
diffRegion := "ap-shanghai"
if diffRegion == kRegion {
diffRegion = "ap-guangzhou"
}
kmsclient, _ := coscrypto.NewKMSClient(c.GetCredential(), diffRegion)
master, _ := coscrypto.CreateMasterKMS(kmsclient, "KMSID", material)
client := coscrypto.NewCryptoClient(c, master)
resp, err := client.Object.Get(context.Background(), name, nil)
assert.Nil(s.T(), resp, "Get Object Failed")
assert.NotNil(s.T(), err, "Get Object Failed")
}
{
// 使用相同的MatDesc和KMSID客户端读取, 期待正确
material := make(map[string]string)
material["desc"] = "cos crypto suite test"
kmsclient, _ := coscrypto.NewKMSClient(c.GetCredential(), kRegion)
master, _ := coscrypto.CreateMasterKMS(kmsclient, os.Getenv("KMSID"), material)
client := coscrypto.NewCryptoClient(c, master)
resp, err := client.Object.Get(context.Background(), name, nil)
assert.Nil(s.T(), err, "Get Object Failed")
defer resp.Body.Close()
decryptedData, _ := ioutil.ReadAll(resp.Body)
assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData")
}
_, err = s.CClient.Object.Delete(context.Background(), name)
assert.Nil(s.T(), err, "DeleteObject Failed")
}

385
crypto/crypto_object_test.go
View File

@ -0,0 +1,385 @@
package coscrypto_test
import (
"bytes"
"context"
"crypto/aes"
"crypto/cipher"
"crypto/md5"
"crypto/rand"
"encoding/base64"
"fmt"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
"github.com/tencentyun/cos-go-sdk-v5"
"github.com/tencentyun/cos-go-sdk-v5/crypto"
"io"
"io/ioutil"
math_rand "math/rand"
"net/http"
"net/url"
"os"
"testing"
"time"
)
const (
kAppid = 1259654469
kBucket = "cosgosdktest-1259654469"
kRegion = "ap-guangzhou"
)
type CosTestSuite struct {
suite.Suite
Client *cos.Client
CClient *coscrypto.CryptoClient
Master coscrypto.MasterCipher
}
func (s *CosTestSuite) SetupSuite() {
u, _ := url.Parse("https://" + kBucket + ".cos." + kRegion + ".myqcloud.com")
b := &cos.BaseURL{BucketURL: u}
s.Client = cos.NewClient(b, &http.Client{
Transport: &cos.AuthorizationTransport{
SecretID: os.Getenv("COS_SECRETID"),
SecretKey: os.Getenv("COS_SECRETKEY"),
},
})
material := make(map[string]string)
material["desc"] = "cos crypto suite test"
kmsclient, _ := coscrypto.NewKMSClient(s.Client.GetCredential(), kRegion)
s.Master, _ = coscrypto.CreateMasterKMS(kmsclient, os.Getenv("KMSID"), material)
s.CClient = coscrypto.NewCryptoClient(s.Client, s.Master)
opt := &cos.BucketPutOptions{
XCosACL: "public-read",
}
r, err := s.Client.Bucket.Put(context.Background(), opt)
if err != nil && r != nil && r.StatusCode == 409 {
fmt.Println("BucketAlreadyOwnedByYou")
} else if err != nil {
assert.Nil(s.T(), err, "PutBucket Failed")
}
}
func (s *CosTestSuite) TestPutGetDeleteObject_DecryptWithKey_10MB() {
name := "test/objectPut" + time.Now().Format(time.RFC3339)
originData := make([]byte, 1024*1024*10+1)
_, err := rand.Read(originData)
f := bytes.NewReader(originData)
// 加密存储
_, err = s.CClient.Object.Put(context.Background(), name, f, nil)
assert.Nil(s.T(), err, "PutObject Failed")
// 获取解密信息
resp, err := s.CClient.Object.Head(context.Background(), name, nil)
assert.Nil(s.T(), err, "HeadObject Failed")
cipherKey := resp.Header.Get(coscrypto.COSClientSideEncryptionKey)
cipherIV := resp.Header.Get(coscrypto.COSClientSideEncryptionStart)
key, err := s.Master.Decrypt([]byte(cipherKey))
assert.Nil(s.T(), err, "Master Decrypt Failed")
iv, err := s.Master.Decrypt([]byte(cipherIV))
assert.Nil(s.T(), err, "Master Decrypt Failed")
// 正常读取
resp, err = s.Client.Object.Get(context.Background(), name, nil)
assert.Nil(s.T(), err, "GetObject Failed")
defer resp.Body.Close()
encryptedData, _ := ioutil.ReadAll(resp.Body)
assert.NotEqual(s.T(), bytes.Compare(encryptedData, originData), 0, "encryptedData == originData")
// 手动解密
block, err := aes.NewCipher(key)
assert.Nil(s.T(), err, "NewCipher Failed")
decrypter := cipher.NewCTR(block, iv)
decryptedData := make([]byte, len(originData))
decrypter.XORKeyStream(decryptedData, encryptedData)
assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData")
_, err = s.CClient.Object.Delete(context.Background(), name)
assert.Nil(s.T(), err, "DeleteObject Failed")
}
func (s *CosTestSuite) TestPutGetDeleteObject_Normal_10MB() {
name := "test/objectPut" + time.Now().Format(time.RFC3339)
originData := make([]byte, 1024*1024*10+1)
_, err := rand.Read(originData)
f := bytes.NewReader(originData)
// 加密存储
_, err = s.CClient.Object.Put(context.Background(), name, f, nil)
assert.Nil(s.T(), err, "PutObject Failed")
// 解密读取
resp, err := s.CClient.Object.Get(context.Background(), name, nil)
assert.Nil(s.T(), err, "GetObject Failed")
defer resp.Body.Close()
decryptedData, _ := ioutil.ReadAll(resp.Body)
assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData")
_, err = s.CClient.Object.Delete(context.Background(), name)
assert.Nil(s.T(), err, "DeleteObject Failed")
}
func (s *CosTestSuite) TestPutGetDeleteObject_ZeroFile() {
name := "test/objectPut" + time.Now().Format(time.RFC3339)
// 加密存储
_, err := s.CClient.Object.Put(context.Background(), name, bytes.NewReader([]byte("")), nil)
assert.Nil(s.T(), err, "PutObject Failed")
// 解密读取
resp, err := s.CClient.Object.Get(context.Background(), name, nil)
assert.Nil(s.T(), err, "GetObject Failed")
defer resp.Body.Close()
decryptedData, _ := ioutil.ReadAll(resp.Body)
assert.Equal(s.T(), bytes.Compare([]byte(""), decryptedData), 0, "decryptData != originData")
_, err = s.CClient.Object.Delete(context.Background(), name)
assert.Nil(s.T(), err, "DeleteObject Failed")
}
func (s *CosTestSuite) TestPutGetDeleteObject_WithMetaData() {
name := "test/objectPut" + time.Now().Format(time.RFC3339)
originData := make([]byte, 1024*1024*10+1)
_, err := rand.Read(originData)
f := bytes.NewReader(originData)
m := md5.New()
m.Write(originData)
contentMD5 := m.Sum(nil)
opt := &cos.ObjectPutOptions{
&cos.ACLHeaderOptions{
XCosACL: "private",
},
&cos.ObjectPutHeaderOptions{
ContentLength: 1024*1024*10 + 1,
ContentMD5: base64.StdEncoding.EncodeToString(contentMD5),
XCosMetaXXX: &http.Header{},
},
}
opt.XCosMetaXXX.Add("x-cos-meta-isEncrypted", "true")
// 加密存储
_, err = s.CClient.Object.Put(context.Background(), name, f, opt)
assert.Nil(s.T(), err, "PutObject Failed")
// 解密读取
resp, err := s.CClient.Object.Get(context.Background(), name, nil)
assert.Nil(s.T(), err, "GetObject Failed")
defer resp.Body.Close()
decryptedData, _ := ioutil.ReadAll(resp.Body)
assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData")
assert.Equal(s.T(), resp.Header.Get("x-cos-meta-isEncrypted"), "true", "meta data isn't consistent")
assert.Equal(s.T(), resp.Header.Get(coscrypto.COSClientSideEncryptionCekAlg), "AES/CTR/NoPadding", "meta data isn't consistent")
assert.Equal(s.T(), resp.Header.Get(coscrypto.COSClientSideEncryptionWrapAlg), "COS/KMS/Crypto", "meta data isn't consistent")
assert.Equal(s.T(), resp.Header.Get(coscrypto.COSClientSideEncryptionUnencryptedContentMD5), base64.StdEncoding.EncodeToString(contentMD5), "meta data isn't consistent")
_, err = s.CClient.Object.Delete(context.Background(), name)
assert.Nil(s.T(), err, "DeleteObject Failed")
}
func (s *CosTestSuite) TestPutGetDeleteObject_ByFile() {
name := "test/objectPut" + time.Now().Format(time.RFC3339)
filepath := "tmpfile" + time.Now().Format(time.RFC3339)
newfile, err := os.Create(filepath)
assert.Nil(s.T(), err, "Create File Failed")
defer os.Remove(filepath)
originData := make([]byte, 1024*1024*10+1)
_, err = rand.Read(originData)
newfile.Write(originData)
newfile.Close()
m := md5.New()
m.Write(originData)
contentMD5 := m.Sum(nil)
opt := &cos.ObjectPutOptions{
&cos.ACLHeaderOptions{
XCosACL: "private",
},
&cos.ObjectPutHeaderOptions{
ContentLength: 1024*1024*10 + 1,
ContentMD5: base64.StdEncoding.EncodeToString(contentMD5),
XCosMetaXXX: &http.Header{},
},
}
opt.XCosMetaXXX.Add("x-cos-meta-isEncrypted", "true")
// 加密存储
_, err = s.CClient.Object.PutFromFile(context.Background(), name, filepath, opt)
assert.Nil(s.T(), err, "PutFromFile Failed")
// 解密读取
downfile := "downfile" + time.Now().Format(time.RFC3339)
resp, err := s.CClient.Object.GetToFile(context.Background(), name, downfile, nil)
assert.Nil(s.T(), err, "GetToFile Failed")
assert.Equal(s.T(), resp.Header.Get("x-cos-meta-isEncrypted"), "true", "meta data isn't consistent")
assert.Equal(s.T(), resp.Header.Get(coscrypto.COSClientSideEncryptionCekAlg), "AES/CTR/NoPadding", "meta data isn't consistent")
assert.Equal(s.T(), resp.Header.Get(coscrypto.COSClientSideEncryptionWrapAlg), "COS/KMS/Crypto", "meta data isn't consistent")
assert.Equal(s.T(), resp.Header.Get(coscrypto.COSClientSideEncryptionUnencryptedContentMD5), base64.StdEncoding.EncodeToString(contentMD5), "meta data isn't consistent")
fd, err := os.Open(downfile)
assert.Nil(s.T(), err, "Open File Failed")
defer os.Remove(downfile)
defer fd.Close()
m = md5.New()
io.Copy(m, fd)
downContentMD5 := m.Sum(nil)
assert.Equal(s.T(), bytes.Compare(contentMD5, downContentMD5), 0, "decryptData != originData")
_, err = s.CClient.Object.Delete(context.Background(), name)
assert.Nil(s.T(), err, "DeleteObject Failed")
}
func (s *CosTestSuite) TestPutGetDeleteObject_DecryptWithNewClient_10MB() {
name := "test/objectPut" + time.Now().Format(time.RFC3339)
originData := make([]byte, 1024*1024*10+1)
_, err := rand.Read(originData)
f := bytes.NewReader(originData)
// 加密存储
_, err = s.CClient.Object.Put(context.Background(), name, f, nil)
assert.Nil(s.T(), err, "PutObject Failed")
u, _ := url.Parse("https://" + kBucket + ".cos." + kRegion + ".myqcloud.com")
b := &cos.BaseURL{BucketURL: u}
c := cos.NewClient(b, &http.Client{
Transport: &cos.AuthorizationTransport{
SecretID: os.Getenv("COS_SECRETID"),
SecretKey: os.Getenv("COS_SECRETKEY"),
},
})
{
// 使用不同的MatDesc客户端读取, 期待错误
material := make(map[string]string)
material["desc"] = "cos crypto suite test 2"
kmsclient, _ := coscrypto.NewKMSClient(c.GetCredential(), kRegion)
master, _ := coscrypto.CreateMasterKMS(kmsclient, os.Getenv("KMSID"), material)
client := coscrypto.NewCryptoClient(c, master)
resp, err := client.Object.Get(context.Background(), name, nil)
assert.Nil(s.T(), resp, "Get Object Failed")
assert.NotNil(s.T(), err, "Get Object Failed")
}
{
// 使用相同的MatDesc客户端读取, 但KMSID不一样，期待正确，kms解密是不需要KMSID
material := make(map[string]string)
material["desc"] = "cos crypto suite test"
kmsclient, _ := coscrypto.NewKMSClient(s.Client.GetCredential(), kRegion)
master, _ := coscrypto.CreateMasterKMS(kmsclient, "KMSID", material)
client := coscrypto.NewCryptoClient(c, master)
resp, err := client.Object.Get(context.Background(), name, nil)
assert.Nil(s.T(), err, "Get Object Failed")
defer resp.Body.Close()
decryptedData, _ := ioutil.ReadAll(resp.Body)
assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData")
}
{
// 使用相同的MatDesc和KMSID客户端读取, 期待正确
material := make(map[string]string)
material["desc"] = "cos crypto suite test"
kmsclient, _ := coscrypto.NewKMSClient(s.Client.GetCredential(), kRegion)
master, _ := coscrypto.CreateMasterKMS(kmsclient, os.Getenv("KMSID"), material)
client := coscrypto.NewCryptoClient(c, master)
resp, err := client.Object.Get(context.Background(), name, nil)
assert.Nil(s.T(), err, "Get Object Failed")
defer resp.Body.Close()
decryptedData, _ := ioutil.ReadAll(resp.Body)
assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData")
}
_, err = s.CClient.Object.Delete(context.Background(), name)
assert.Nil(s.T(), err, "DeleteObject Failed")
}
func (s *CosTestSuite) TestPutGetDeleteObject_RangeGet() {
name := "test/objectPut" + time.Now().Format(time.RFC3339)
contentLength := 1024*1024*10 + 1
originData := make([]byte, contentLength)
_, err := rand.Read(originData)
f := bytes.NewReader(originData)
// 加密存储
_, err = s.CClient.Object.Put(context.Background(), name, f, nil)
assert.Nil(s.T(), err, "PutObject Failed")
// Range解密读取
for i := 0; i < 10; i++ {
math_rand.Seed(time.Now().UnixNano())
rangeStart := math_rand.Intn(contentLength)
rangeEnd := rangeStart + math_rand.Intn(contentLength-rangeStart)
if rangeEnd == rangeStart || rangeStart >= contentLength-1 {
continue
}
opt := &cos.ObjectGetOptions{
Range: fmt.Sprintf("bytes=%v-%v", rangeStart, rangeEnd),
}
resp, err := s.CClient.Object.Get(context.Background(), name, opt)
assert.Nil(s.T(), err, "GetObject Failed")
defer resp.Body.Close()
decryptedData, _ := ioutil.ReadAll(resp.Body)
assert.Equal(s.T(), bytes.Compare(originData[rangeStart:rangeEnd+1], decryptedData), 0, "decryptData != originData")
}
// 解密读取
resp, err := s.CClient.Object.Get(context.Background(), name, nil)
assert.Nil(s.T(), err, "GetObject Failed")
defer resp.Body.Close()
decryptedData, _ := ioutil.ReadAll(resp.Body)
assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData")
_, err = s.CClient.Object.Delete(context.Background(), name)
assert.Nil(s.T(), err, "DeleteObject Failed")
}
func (s *CosTestSuite) TestPutGetDeleteObject_WithListenerAndRange() {
name := "test/objectPut" + time.Now().Format(time.RFC3339)
contentLength := 1024*1024*10 + 1
originData := make([]byte, contentLength)
_, err := rand.Read(originData)
f := bytes.NewReader(originData)
// 加密存储
popt := &cos.ObjectPutOptions{
nil,
&cos.ObjectPutHeaderOptions{
Listener: &cos.DefaultProgressListener{},
},
}
_, err = s.CClient.Object.Put(context.Background(), name, f, popt)
assert.Nil(s.T(), err, "PutObject Failed")
// Range解密读取
for i := 0; i < 10; i++ {
math_rand.Seed(time.Now().UnixNano())
rangeStart := math_rand.Intn(contentLength)
rangeEnd := rangeStart + math_rand.Intn(contentLength-rangeStart)
if rangeEnd == rangeStart || rangeStart >= contentLength-1 {
continue
}
opt := &cos.ObjectGetOptions{
Range: fmt.Sprintf("bytes=%v-%v", rangeStart, rangeEnd),
Listener: &cos.DefaultProgressListener{},
}
resp, err := s.CClient.Object.Get(context.Background(), name, opt)
assert.Nil(s.T(), err, "GetObject Failed")
defer resp.Body.Close()
decryptedData, _ := ioutil.ReadAll(resp.Body)
assert.Equal(s.T(), bytes.Compare(originData[rangeStart:rangeEnd+1], decryptedData), 0, "decryptData != originData")
}
// 解密读取
opt := &cos.ObjectGetOptions{
Listener: &cos.DefaultProgressListener{},
}
resp, err := s.CClient.Object.Get(context.Background(), name, opt)
assert.Nil(s.T(), err, "GetObject Failed")
defer resp.Body.Close()
decryptedData, _ := ioutil.ReadAll(resp.Body)
assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData")
_, err = s.CClient.Object.Delete(context.Background(), name)
assert.Nil(s.T(), err, "DeleteObject Failed")
}
func TestCosTestSuite(t *testing.T) {
suite.Run(t, new(CosTestSuite))
}
func (s *CosTestSuite) TearDownSuite() {
}

141
crypto/crypto_type.go
View File

@ -0,0 +1,141 @@
package coscrypto
import (
"crypto/rand"
"encoding/binary"
"fmt"
"io"
math_rand "math/rand"
"time"
)
const (
COSClientSideEncryptionKey string = "x-cos-meta-client-side-encryption-key"
COSClientSideEncryptionStart = "x-cos-meta-client-side-encryption-start"
COSClientSideEncryptionCekAlg = "x-cos-meta-client-side-encryption-cek-alg"
COSClientSideEncryptionWrapAlg = "x-cos-meta-client-side-encryption-wrap-alg"
COSClientSideEncryptionMatDesc = "x-cos-meta-client-side-encryption-matdesc"
COSClientSideEncryptionUnencryptedContentLength = "x-cos-meta-client-side-encryption-unencrypted-content-length"
COSClientSideEncryptionUnencryptedContentMD5 = "x-cos-meta-client-side-encryption-unencrypted-content-md5"
COSClientSideEncryptionDataSize = "x-cos-meta-client-side-encryption-data-size"
COSClientSideEncryptionPartSize = "x-cos-meta-client-side-encryption-part-size"
COSClientUserAgent = "User-Agent"
)
const (
CosKmsCryptoWrap = "COS/KMS/Crypto"
AesCtrAlgorithm = "AES/CTR/NoPadding"
EncryptionUaSuffix = "COSEncryptionClient"
)
type MasterCipher interface {
Encrypt([]byte) ([]byte, error)
Decrypt([]byte) ([]byte, error)
GetWrapAlgorithm() string
GetMatDesc() string
}
type ContentCipherBuilder interface {
ContentCipher() (ContentCipher, error)
ContentCipherEnv(Envelope) (ContentCipher, error)
GetMatDesc() string
}
type ContentCipher interface {
EncryptContent(io.Reader) (io.ReadCloser, error)
DecryptContent(io.Reader) (io.ReadCloser, error)
Clone(cd CipherData) (ContentCipher, error)
GetEncryptedLen(int64) int64
GetCipherData() *CipherData
GetAlignLen() int
}
type Envelope struct {
IV string
CipherKey string
MatDesc string
WrapAlg string
CEKAlg string
UnencryptedMD5 string
UnencryptedContentLen string
}
func (el Envelope) IsValid() bool {
return len(el.IV) > 0 &&
len(el.CipherKey) > 0 &&
len(el.WrapAlg) > 0 &&
len(el.CEKAlg) > 0
}
func (el Envelope) String() string {
return fmt.Sprintf("IV=%s&CipherKey=%s&WrapAlg=%s&CEKAlg=%s", el.IV, el.CipherKey, el.WrapAlg, el.CEKAlg)
}
type CipherData struct {
IV []byte
Key []byte
MatDesc string
WrapAlgorithm string
CEKAlgorithm string
EncryptedIV []byte
EncryptedKey []byte
}
func (cd *CipherData) RandomKeyIv(keyLen int, ivLen int) error {
math_rand.Seed(time.Now().UnixNano())
// Key
cd.Key = make([]byte, keyLen)
if _, err := io.ReadFull(rand.Reader, cd.Key); err != nil {
return err
}
// sizeof uint64
if ivLen < 8 {
return fmt.Errorf("ivLen:%d less than 8", ivLen)
}
// IV: | nonce: 8 bytes | Serial number: 8 bytes |
cd.IV = make([]byte, ivLen)
if _, err := io.ReadFull(rand.Reader, cd.IV[0:ivLen-8]); err != nil {
return err
}
// only use 4 byte,in order not to overflow when SeekIV()
randNumber := math_rand.Uint32()
cd.SetIV(uint64(randNumber))
return nil
}
func (cd *CipherData) SetIV(iv uint64) {
ivLen := len(cd.IV)
binary.BigEndian.PutUint64(cd.IV[ivLen-8:], iv)
}
func (cd *CipherData) GetIV() uint64 {
ivLen := len(cd.IV)
return binary.BigEndian.Uint64(cd.IV[ivLen-8:])
}
func (cd *CipherData) SeekIV(startPos uint64) {
cd.SetIV(cd.GetIV() + startPos/uint64(len(cd.IV)))
}
func (cd *CipherData) Clone() CipherData {
var cloneCd CipherData
cloneCd = *cd
cloneCd.Key = make([]byte, len(cd.Key))
copy(cloneCd.Key, cd.Key)
cloneCd.IV = make([]byte, len(cd.IV))
copy(cloneCd.IV, cd.IV)
cloneCd.EncryptedIV = make([]byte, len(cd.EncryptedIV))
copy(cloneCd.EncryptedIV, cd.EncryptedIV)
cloneCd.EncryptedKey = make([]byte, len(cd.EncryptedKey))
copy(cloneCd.EncryptedKey, cd.EncryptedKey)
return cloneCd
}

87
crypto/master_kms_cipher.go
View File

@ -0,0 +1,87 @@
package coscrypto
import (
"encoding/base64"
"encoding/json"
"fmt"
"github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common"
"github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/profile"
kms "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/kms/v20190118"
"github.com/tencentyun/cos-go-sdk-v5"
)
const (
KMSEndPoint = "kms.tencentcloudapi.com"
)
type MasterKMSCipher struct {
Client *kms.Client
KmsId string
MatDesc string
}
func NewKMSClient(cred *cos.Credential, region string) (*kms.Client, error) {
if cred == nil {
fmt.Errorf("credential is nil")
}
credential := common.NewTokenCredential(
cred.SecretID,
cred.SecretKey,
cred.SessionToken,
)
cpf := profile.NewClientProfile()
cpf.HttpProfile.Endpoint = KMSEndPoint
client, err := kms.NewClient(credential, region, cpf)
return client, err
}
func CreateMasterKMS(client *kms.Client, kmsId string, desc map[string]string) (MasterCipher, error) {
if kmsId == "" || client == nil {
return nil, fmt.Errorf("KMS ID is empty or kms client is nil")
}
var kmsCipher MasterKMSCipher
var jdesc string
if len(desc) > 0 {
bs, err := json.Marshal(desc)
if err != nil {
return nil, err
}
jdesc = string(bs)
}
kmsCipher.Client = client
kmsCipher.KmsId = kmsId
kmsCipher.MatDesc = jdesc
return &kmsCipher, nil
}
func (kc *MasterKMSCipher) Encrypt(plaintext []byte) ([]byte, error) {
request := kms.NewEncryptRequest()
request.KeyId = common.StringPtr(kc.KmsId)
request.EncryptionContext = common.StringPtr(kc.MatDesc)
request.Plaintext = common.StringPtr(base64.StdEncoding.EncodeToString(plaintext))
resp, err := kc.Client.Encrypt(request)
if err != nil {
return nil, err
}
return []byte(*resp.Response.CiphertextBlob), nil
}
func (kc *MasterKMSCipher) Decrypt(ciphertext []byte) ([]byte, error) {
request := kms.NewDecryptRequest()
request.CiphertextBlob = common.StringPtr(string(ciphertext))
request.EncryptionContext = common.StringPtr(kc.MatDesc)
resp, err := kc.Client.Decrypt(request)
if err != nil {
return nil, err
}
return base64.StdEncoding.DecodeString(*resp.Response.Plaintext)
}
func (kc *MasterKMSCipher) GetWrapAlgorithm() string {
return CosKmsCryptoWrap
}
func (kc *MasterKMSCipher) GetMatDesc() string {
return kc.MatDesc
}

91
crypto/master_kms_cipher_test.go
View File

@ -0,0 +1,91 @@
package coscrypto_test
import (
"bytes"
"crypto/rand"
"encoding/base64"
"encoding/json"
"github.com/stretchr/testify/assert"
"github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common"
kms "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/kms/v20190118"
"github.com/tencentyun/cos-go-sdk-v5"
"github.com/tencentyun/cos-go-sdk-v5/crypto"
"os"
)
func (s *CosTestSuite) TestMasterKmsCipher_TestKmsClient() {
kmsclient, _ := coscrypto.NewKMSClient(&cos.Credential{
SecretID: os.Getenv("COS_SECRETID"),
SecretKey: os.Getenv("COS_SECRETKEY"),
}, kRegion)
originData := make([]byte, 1024)
_, err := rand.Read(originData)
ctx := make(map[string]string)
ctx["desc"] = string(originData[:10])
bs, _ := json.Marshal(ctx)
ctxJson := string(bs)
enReq := kms.NewEncryptRequest()
enReq.KeyId = common.StringPtr(os.Getenv("KMSID"))
enReq.EncryptionContext = common.StringPtr(ctxJson)
enReq.Plaintext = common.StringPtr(base64.StdEncoding.EncodeToString(originData))
enResp, err := kmsclient.Encrypt(enReq)
assert.Nil(s.T(), err, "Encrypt Failed")
encryptedData := []byte(*enResp.Response.CiphertextBlob)
deReq := kms.NewDecryptRequest()
deReq.CiphertextBlob = common.StringPtr(string(encryptedData))
deReq.EncryptionContext = common.StringPtr(ctxJson)
deResp, err := kmsclient.Decrypt(deReq)
assert.Nil(s.T(), err, "Decrypt Failed")
decryptedData, err := base64.StdEncoding.DecodeString(*deResp.Response.Plaintext)
assert.Nil(s.T(), err, "base64 Decode Failed")
assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "originData != decryptedData")
}
func (s *CosTestSuite) TestMasterKmsCipher_TestNormal() {
kmsclient, _ := coscrypto.NewKMSClient(&cos.Credential{
SecretID: os.Getenv("COS_SECRETID"),
SecretKey: os.Getenv("COS_SECRETKEY"),
}, kRegion)
desc := make(map[string]string)
desc["test"] = "TestMasterKmsCipher_TestNormal"
master, err := coscrypto.CreateMasterKMS(kmsclient, os.Getenv("KMSID"), desc)
assert.Nil(s.T(), err, "CreateMasterKMS Failed")
originData := make([]byte, 1024)
_, err = rand.Read(originData)
encryptedData, err := master.Encrypt(originData)
assert.Nil(s.T(), err, "Encrypt Failed")
decryptedData, err := master.Decrypt(encryptedData)
assert.Nil(s.T(), err, "Decrypt Failed")
assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "originData != decryptedData")
}
func (s *CosTestSuite) TestMasterKmsCipher_TestError() {
kmsclient, _ := coscrypto.NewKMSClient(&cos.Credential{
SecretID: os.Getenv("COS_SECRETID"),
SecretKey: os.Getenv("COS_SECRETKEY"),
}, kRegion)
desc := make(map[string]string)
desc["test"] = "TestMasterKmsCipher_TestNormal"
master, err := coscrypto.CreateMasterKMS(kmsclient, "ErrorKMSID", desc)
assert.Nil(s.T(), err, "CreateMasterKMS Failed")
originData := make([]byte, 1024)
_, err = rand.Read(originData)
encryptedData, err := master.Encrypt(originData)
assert.NotNil(s.T(), err, "Encrypt Failed")
decryptedData, err := master.Decrypt(encryptedData)
assert.NotNil(s.T(), err, "Decrypt Failed")
assert.NotEqual(s.T(), bytes.Compare(originData, decryptedData), 0, "originData != decryptedData")
}

305
example/crypto/crypto_sample.go
View File

@ -0,0 +1,305 @@
package main
import (
"bytes"
"context"
"crypto/md5"
"crypto/rand"
"fmt"
"io"
"io/ioutil"
math_rand "math/rand"
"net/http"
"net/url"
"os"
"time"
"github.com/tencentyun/cos-go-sdk-v5"
"github.com/tencentyun/cos-go-sdk-v5/crypto"
"github.com/tencentyun/cos-go-sdk-v5/debug"
)
func log_status(err error) {
if err == nil {
return
}
if cos.IsNotFoundError(err) {
// WARN
fmt.Println("WARN: Resource is not existed")
} else if e, ok := cos.IsCOSError(err); ok {
fmt.Printf("ERROR: Code: %v\n", e.Code)
fmt.Printf("ERROR: Message: %v\n", e.Message)
fmt.Printf("ERROR: Resource: %v\n", e.Resource)
fmt.Printf("ERROR: RequestId: %v\n", e.RequestID)
// ERROR
} else {
fmt.Printf("ERROR: %v\n", err)
// ERROR
}
os.Exit(1)
}
func simple_put_object() {
u, _ := url.Parse("https://test-1259654469.cos.ap-guangzhou.myqcloud.com")
b := &cos.BaseURL{BucketURL: u}
c := cos.NewClient(b, &http.Client{
Transport: &cos.AuthorizationTransport{
SecretID: os.Getenv("COS_SECRETID"),
SecretKey: os.Getenv("COS_SECRETKEY"),
Transport: &debug.DebugRequestTransport{
RequestHeader: true,
RequestBody: false,
ResponseHeader: true,
ResponseBody: false,
},
},
})
// Case1 上传对象
name := "test/example2"
fmt.Println("============== simple_put_object ======================")
// 该标识信息唯一确认一个主加密密钥, 解密时，需要传入相同的标识信息
materialDesc := make(map[string]string)
materialDesc["desc"] = "<material information of your master encrypt key>"
// 创建KMS客户端
kmsclient, _ := coscrypto.NewKMSClient(c.GetCredential(), "ap-guangzhou")
// 创建KMS主加密密钥，标识信息和主密钥一一对应
kmsID := os.Getenv("KMSID")
masterCipher, _ := coscrypto.CreateMasterKMS(kmsclient, kmsID, materialDesc)
// 创建加密客户端
client := coscrypto.NewCryptoClient(c, masterCipher)
contentLength := 1024*1024*10 + 1
originData := make([]byte, contentLength)
_, err := rand.Read(originData)
f := bytes.NewReader(originData)
// 加密上传
_, err = client.Object.Put(context.Background(), name, f, nil)
log_status(err)
math_rand.Seed(time.Now().UnixNano())
rangeStart := math_rand.Intn(contentLength)
rangeEnd := rangeStart + math_rand.Intn(contentLength-rangeStart)
opt := &cos.ObjectGetOptions{
Range: fmt.Sprintf("bytes=%v-%v", rangeStart, rangeEnd),
}
// 解密下载
resp, err := client.Object.Get(context.Background(), name, opt)
log_status(err)
defer resp.Body.Close()
decryptedData, _ := ioutil.ReadAll(resp.Body)
if bytes.Compare(decryptedData, originData[rangeStart:rangeEnd+1]) != 0 {
fmt.Println("Error: encryptedData != originData")
}
}
func simple_put_object_from_file() {
u, _ := url.Parse("https://test-1259654469.cos.ap-guangzhou.myqcloud.com")
b := &cos.BaseURL{BucketURL: u}
c := cos.NewClient(b, &http.Client{
Transport: &cos.AuthorizationTransport{
SecretID: os.Getenv("COS_SECRETID"),
SecretKey: os.Getenv("COS_SECRETKEY"),
Transport: &debug.DebugRequestTransport{
RequestHeader: true,
RequestBody: false,
ResponseHeader: true,
ResponseBody: false,
},
},
})
// Case1 上传对象
name := "test/example1"
fmt.Println("============== simple_put_object_from_file ======================")
// 该标识信息唯一确认一个主加密密钥, 解密时，需要传入相同的标识信息
materialDesc := make(map[string]string)
materialDesc["desc"] = "<material information of your master encrypt key>"
// 创建KMS客户端
kmsclient, _ := coscrypto.NewKMSClient(c.GetCredential(), "ap-guangzhou")
// 创建KMS主加密密钥，标识信息和主密钥一一对应
kmsID := os.Getenv("KMSID")
masterCipher, _ := coscrypto.CreateMasterKMS(kmsclient, kmsID, materialDesc)
// 创建加密客户端
client := coscrypto.NewCryptoClient(c, masterCipher)
filepath := "test"
fd, err := os.Open(filepath)
log_status(err)
defer fd.Close()
m := md5.New()
io.Copy(m, fd)
originDataMD5 := m.Sum(nil)
// 加密上传
_, err = client.Object.PutFromFile(context.Background(), name, filepath, nil)
log_status(err)
// 解密下载
_, err = client.Object.GetToFile(context.Background(), name, "./test.download", nil)
log_status(err)
fd, err = os.Open("./test.download")
log_status(err)
defer fd.Close()
m = md5.New()
io.Copy(m, fd)
decryptedDataMD5 := m.Sum(nil)
if bytes.Compare(decryptedDataMD5, originDataMD5) != 0 {
fmt.Println("Error: encryptedData != originData")
}
}
func multi_put_object() {
u, _ := url.Parse("https://test-1259654469.cos.ap-guangzhou.myqcloud.com")
b := &cos.BaseURL{BucketURL: u}
c := cos.NewClient(b, &http.Client{
Transport: &cos.AuthorizationTransport{
SecretID: os.Getenv("COS_SECRETID"),
SecretKey: os.Getenv("COS_SECRETKEY"),
Transport: &debug.DebugRequestTransport{
RequestHeader: true,
RequestBody: false,
ResponseHeader: true,
ResponseBody: false,
},
},
})
// Case1 上传对象
name := "test/example1"
fmt.Println("============== multi_put_object ======================")
// 该标识信息唯一确认一个主加密密钥, 解密时，需要传入相同的标识信息
materialDesc := make(map[string]string)
materialDesc["desc"] = "<material information of your master encrypt key>"
// 创建KMS客户端
kmsclient, _ := coscrypto.NewKMSClient(c.GetCredential(), "ap-guangzhou")
// 创建KMS主加密密钥，标识信息和主密钥一一对应
kmsID := os.Getenv("KMSID")
masterCipher, _ := coscrypto.CreateMasterKMS(kmsclient, kmsID, materialDesc)
// 创建加密客户端
client := coscrypto.NewCryptoClient(c, masterCipher)
contentLength := int64(1024*1024*10 + 1)
originData := make([]byte, contentLength)
_, err := rand.Read(originData)
log_status(err)
// 分块上传
cryptoCtx := coscrypto.CryptoContext{
DataSize: contentLength,
// 每个分块需要16字节对齐
PartSize: (contentLength / 16 / 3) * 16,
}
v, _, err := client.Object.InitiateMultipartUpload(context.Background(), name, nil, &cryptoCtx)
log_status(err)
// 切分数据
chunks, _, err := cos.SplitSizeIntoChunks(contentLength, cryptoCtx.PartSize)
log_status(err)
optcom := &cos.CompleteMultipartUploadOptions{}
for _, chunk := range chunks {
opt := &cos.ObjectUploadPartOptions{
ContentLength: chunk.Size,
}
f := bytes.NewReader(originData[chunk.OffSet : chunk.OffSet+chunk.Size])
resp, err := client.Object.UploadPart(context.Background(), name, v.UploadID, chunk.Number, f, opt, &cryptoCtx)
log_status(err)
optcom.Parts = append(optcom.Parts, cos.Object{
PartNumber: chunk.Number, ETag: resp.Header.Get("ETag"),
})
}
_, _, err = client.Object.CompleteMultipartUpload(context.Background(), name, v.UploadID, optcom)
log_status(err)
resp, err := client.Object.Get(context.Background(), name, nil)
log_status(err)
defer resp.Body.Close()
decryptedData, _ := ioutil.ReadAll(resp.Body)
if bytes.Compare(decryptedData, originData) != 0 {
fmt.Println("Error: encryptedData != originData")
}
}
func multi_put_object_from_file() {
u, _ := url.Parse("https://test-1259654469.cos.ap-guangzhou.myqcloud.com")
b := &cos.BaseURL{BucketURL: u}
c := cos.NewClient(b, &http.Client{
Transport: &cos.AuthorizationTransport{
SecretID: os.Getenv("COS_SECRETID"),
SecretKey: os.Getenv("COS_SECRETKEY"),
Transport: &debug.DebugRequestTransport{
RequestHeader: true,
RequestBody: false,
ResponseHeader: true,
ResponseBody: false,
},
},
})
// Case1 上传对象
name := "test/example1"
fmt.Println("============== multi_put_object_from_file ======================")
// 该标识信息唯一确认一个主加密密钥, 解密时，需要传入相同的标识信息
materialDesc := make(map[string]string)
materialDesc["desc"] = "<material information of your master encrypt key>"
// 创建KMS客户端
kmsclient, _ := coscrypto.NewKMSClient(c.GetCredential(), "ap-guangzhou")
// 创建KMS主加密密钥，标识信息和主密钥一一对应
kmsID := os.Getenv("KMSID")
masterCipher, _ := coscrypto.CreateMasterKMS(kmsclient, kmsID, materialDesc)
// 创建加密客户端
client := coscrypto.NewCryptoClient(c, masterCipher)
filepath := "test"
stat, err := os.Stat(filepath)
log_status(err)
contentLength := stat.Size()
// 分块上传
cryptoCtx := coscrypto.CryptoContext{
DataSize: contentLength,
// 每个分块需要16字节对齐
PartSize: (contentLength / 16 / 3) * 16,
}
// 切分数据
_, chunks, _, err := cos.SplitFileIntoChunks(filepath, cryptoCtx.PartSize)
log_status(err)
// init mulitupload
v, _, err := client.Object.InitiateMultipartUpload(context.Background(), name, nil, &cryptoCtx)
log_status(err)
// part upload
optcom := &cos.CompleteMultipartUploadOptions{}
for _, chunk := range chunks {
fd, err := os.Open(filepath)
log_status(err)
opt := &cos.ObjectUploadPartOptions{
ContentLength: chunk.Size,
}
fd.Seek(chunk.OffSet, os.SEEK_SET)
resp, err := client.Object.UploadPart(context.Background(), name, v.UploadID, chunk.Number, io.LimitReader(fd, chunk.Size), opt, &cryptoCtx)
log_status(err)
optcom.Parts = append(optcom.Parts, cos.Object{
PartNumber: chunk.Number, ETag: resp.Header.Get("ETag"),
})
}
// complete upload
_, _, err = client.Object.CompleteMultipartUpload(context.Background(), name, v.UploadID, optcom)
log_status(err)
_, err = client.Object.GetToFile(context.Background(), name, "test.download", nil)
log_status(err)
}
func main() {
simple_put_object()
simple_put_object_from_file()
multi_put_object()
multi_put_object_from_file()
}
Write Preview
Loading…
Cancel
Save