You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

387 lines
14 KiB

4 years ago
4 years ago
  1. package coscrypto_test
  2. import (
  3. "bytes"
  4. "context"
  5. "crypto/aes"
  6. "crypto/cipher"
  7. "crypto/md5"
  8. "crypto/rand"
  9. "encoding/base64"
  10. "fmt"
  11. "github.com/stretchr/testify/assert"
  12. "github.com/tencentyun/cos-go-sdk-v5"
  13. "github.com/tencentyun/cos-go-sdk-v5/crypto"
  14. "io"
  15. "io/ioutil"
  16. math_rand "math/rand"
  17. "net/http"
  18. "net/url"
  19. "os"
  20. "sort"
  21. "sync"
  22. "time"
  23. )
  24. func (s *CosTestSuite) TestMultiUpload_Normal() {
  25. name := "test/ObjectPut" + time.Now().Format(time.RFC3339)
  26. contentLength := int64(1024*1024*10 + 1)
  27. originData := make([]byte, contentLength)
  28. _, err := rand.Read(originData)
  29. cryptoCtx := coscrypto.CryptoContext{
  30. DataSize: contentLength,
  31. PartSize: (contentLength / 16 / 3) * 16,
  32. }
  33. v, _, err := s.CClient.Object.InitiateMultipartUpload(context.Background(), name, nil, &cryptoCtx)
  34. assert.Nil(s.T(), err, "Init Failed")
  35. chunks, _, err := cos.SplitSizeIntoChunks(contentLength, cryptoCtx.PartSize)
  36. assert.Nil(s.T(), err, "Split Failed")
  37. optcom := &cos.CompleteMultipartUploadOptions{}
  38. for _, chunk := range chunks {
  39. opt := &cos.ObjectUploadPartOptions{
  40. ContentLength: chunk.Size,
  41. }
  42. f := bytes.NewReader(originData[chunk.OffSet : chunk.OffSet+chunk.Size])
  43. resp, err := s.CClient.Object.UploadPart(context.Background(), name, v.UploadID, chunk.Number, io.LimitReader(f, chunk.Size), opt, &cryptoCtx)
  44. assert.Nil(s.T(), err, "UploadPart failed")
  45. optcom.Parts = append(optcom.Parts, cos.Object{
  46. PartNumber: chunk.Number, ETag: resp.Header.Get("ETag"),
  47. })
  48. }
  49. _, _, err = s.CClient.Object.CompleteMultipartUpload(context.Background(), name, v.UploadID, optcom)
  50. assert.Nil(s.T(), err, "Complete Failed")
  51. resp, err := s.CClient.Object.Get(context.Background(), name, nil)
  52. assert.Nil(s.T(), err, "GetObject Failed")
  53. defer resp.Body.Close()
  54. decryptedData, _ := ioutil.ReadAll(resp.Body)
  55. assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData")
  56. _, err = s.CClient.Object.Delete(context.Background(), name)
  57. assert.Nil(s.T(), err, "DeleteObject Failed")
  58. }
  59. func (s *CosTestSuite) TestMultiUpload_DecryptWithKey() {
  60. name := "test/ObjectPut" + time.Now().Format(time.RFC3339)
  61. contentLength := int64(1024*1024*10 + 1)
  62. originData := make([]byte, contentLength)
  63. _, err := rand.Read(originData)
  64. f := bytes.NewReader(originData)
  65. // 分块上传
  66. cryptoCtx := coscrypto.CryptoContext{
  67. DataSize: contentLength,
  68. PartSize: (contentLength / 16 / 3) * 16,
  69. }
  70. v, _, err := s.CClient.Object.InitiateMultipartUpload(context.Background(), name, nil, &cryptoCtx)
  71. assert.Nil(s.T(), err, "Init Failed")
  72. chunks, _, err := cos.SplitSizeIntoChunks(contentLength, cryptoCtx.PartSize)
  73. assert.Nil(s.T(), err, "Split Failed")
  74. optcom := &cos.CompleteMultipartUploadOptions{}
  75. for _, chunk := range chunks {
  76. opt := &cos.ObjectUploadPartOptions{
  77. ContentLength: chunk.Size,
  78. Listener: &cos.DefaultProgressListener{},
  79. }
  80. resp, err := s.CClient.Object.UploadPart(context.Background(), name, v.UploadID, chunk.Number, io.LimitReader(f, chunk.Size), opt, &cryptoCtx)
  81. assert.Nil(s.T(), err, "UploadPart failed")
  82. optcom.Parts = append(optcom.Parts, cos.Object{
  83. PartNumber: chunk.Number, ETag: resp.Header.Get("ETag"),
  84. })
  85. }
  86. _, _, err = s.CClient.Object.CompleteMultipartUpload(context.Background(), name, v.UploadID, optcom)
  87. assert.Nil(s.T(), err, "Complete Failed")
  88. // 正常读取
  89. resp, err := s.Client.Object.Get(context.Background(), name, nil)
  90. assert.Nil(s.T(), err, "GetObject Failed")
  91. defer resp.Body.Close()
  92. encryptedData, _ := ioutil.ReadAll(resp.Body)
  93. assert.NotEqual(s.T(), bytes.Compare(encryptedData, originData), 0, "encryptedData == originData")
  94. // 获取解密信息
  95. resp, err = s.CClient.Object.Head(context.Background(), name, nil)
  96. assert.Nil(s.T(), err, "HeadObject Failed")
  97. cipherKey := resp.Header.Get(coscrypto.COSClientSideEncryptionKey)
  98. cipherIV := resp.Header.Get(coscrypto.COSClientSideEncryptionStart)
  99. key, err := s.Master.Decrypt([]byte(cipherKey))
  100. assert.Nil(s.T(), err, "Master Decrypt Failed")
  101. iv, err := s.Master.Decrypt([]byte(cipherIV))
  102. assert.Nil(s.T(), err, "Master Decrypt Failed")
  103. // 手动解密
  104. block, err := aes.NewCipher(key)
  105. assert.Nil(s.T(), err, "NewCipher Failed")
  106. decrypter := cipher.NewCTR(block, iv)
  107. decryptedData := make([]byte, len(originData))
  108. decrypter.XORKeyStream(decryptedData, encryptedData)
  109. assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData")
  110. _, err = s.CClient.Object.Delete(context.Background(), name)
  111. assert.Nil(s.T(), err, "DeleteObject Failed")
  112. }
  113. func (s *CosTestSuite) TestMultiUpload_PutFromFile() {
  114. name := "test/ObjectPut" + time.Now().Format(time.RFC3339)
  115. filepath := "tmpfile" + time.Now().Format(time.RFC3339)
  116. newfile, err := os.Create(filepath)
  117. assert.Nil(s.T(), err, "Create File Failed")
  118. defer os.Remove(filepath)
  119. contentLength := int64(1024*1024*10 + 1)
  120. originData := make([]byte, contentLength)
  121. _, err = rand.Read(originData)
  122. newfile.Write(originData)
  123. newfile.Close()
  124. m := md5.New()
  125. m.Write(originData)
  126. contentMD5 := m.Sum(nil)
  127. cryptoCtx := coscrypto.CryptoContext{
  128. DataSize: contentLength,
  129. PartSize: (contentLength / 16 / 3) * 16,
  130. }
  131. v, _, err := s.CClient.Object.InitiateMultipartUpload(context.Background(), name, nil, &cryptoCtx)
  132. assert.Nil(s.T(), err, "Init Failed")
  133. _, chunks, _, err := cos.SplitFileIntoChunks(filepath, cryptoCtx.PartSize)
  134. assert.Nil(s.T(), err, "Split Failed")
  135. optcom := &cos.CompleteMultipartUploadOptions{}
  136. var wg sync.WaitGroup
  137. var mtx sync.Mutex
  138. for _, chunk := range chunks {
  139. wg.Add(1)
  140. go func(chk cos.Chunk) {
  141. defer wg.Done()
  142. fd, err := os.Open(filepath)
  143. assert.Nil(s.T(), err, "Open File Failed")
  144. opt := &cos.ObjectUploadPartOptions{
  145. ContentLength: chk.Size,
  146. }
  147. fd.Seek(chk.OffSet, os.SEEK_SET)
  148. resp, err := s.CClient.Object.UploadPart(context.Background(), name, v.UploadID, chk.Number, io.LimitReader(fd, chk.Size), opt, &cryptoCtx)
  149. assert.Nil(s.T(), err, "UploadPart failed")
  150. mtx.Lock()
  151. optcom.Parts = append(optcom.Parts, cos.Object{
  152. PartNumber: chk.Number, ETag: resp.Header.Get("ETag"),
  153. })
  154. mtx.Unlock()
  155. }(chunk)
  156. }
  157. wg.Wait()
  158. sort.Sort(cos.ObjectList(optcom.Parts))
  159. _, _, err = s.CClient.Object.CompleteMultipartUpload(context.Background(), name, v.UploadID, optcom)
  160. assert.Nil(s.T(), err, "Complete Failed")
  161. downfile := "downfile" + time.Now().Format(time.RFC3339)
  162. _, err = s.CClient.Object.GetToFile(context.Background(), name, downfile, nil)
  163. assert.Nil(s.T(), err, "GetObject Failed")
  164. m = md5.New()
  165. fd, err := os.Open(downfile)
  166. assert.Nil(s.T(), err, "Open File Failed")
  167. defer os.Remove(downfile)
  168. defer fd.Close()
  169. io.Copy(m, fd)
  170. downContentMD5 := m.Sum(nil)
  171. assert.Equal(s.T(), bytes.Compare(contentMD5, downContentMD5), 0, "decryptData != originData")
  172. _, err = s.CClient.Object.Delete(context.Background(), name)
  173. assert.Nil(s.T(), err, "DeleteObject Failed")
  174. }
  175. func (s *CosTestSuite) TestMultiUpload_GetWithRange() {
  176. name := "test/ObjectPut" + time.Now().Format(time.RFC3339)
  177. filepath := "tmpfile" + time.Now().Format(time.RFC3339)
  178. newfile, err := os.Create(filepath)
  179. assert.Nil(s.T(), err, "Create File Failed")
  180. defer os.Remove(filepath)
  181. contentLength := int64(1024*1024*10 + 1)
  182. originData := make([]byte, contentLength)
  183. _, err = rand.Read(originData)
  184. newfile.Write(originData)
  185. newfile.Close()
  186. m := md5.New()
  187. m.Write(originData)
  188. contentMD5 := m.Sum(nil)
  189. cryptoCtx := coscrypto.CryptoContext{
  190. DataSize: contentLength,
  191. PartSize: (contentLength / 16 / 3) * 16,
  192. }
  193. iniopt := &cos.InitiateMultipartUploadOptions{
  194. &cos.ACLHeaderOptions{
  195. XCosACL: "private",
  196. },
  197. &cos.ObjectPutHeaderOptions{
  198. ContentMD5: base64.StdEncoding.EncodeToString(contentMD5),
  199. XCosMetaXXX: &http.Header{},
  200. },
  201. }
  202. iniopt.XCosMetaXXX.Add("x-cos-meta-isEncrypted", "true")
  203. v, _, err := s.CClient.Object.InitiateMultipartUpload(context.Background(), name, iniopt, &cryptoCtx)
  204. assert.Nil(s.T(), err, "Init Failed")
  205. _, chunks, _, err := cos.SplitFileIntoChunks(filepath, cryptoCtx.PartSize)
  206. assert.Nil(s.T(), err, "Split Failed")
  207. optcom := &cos.CompleteMultipartUploadOptions{}
  208. var wg sync.WaitGroup
  209. var mtx sync.Mutex
  210. for _, chunk := range chunks {
  211. wg.Add(1)
  212. go func(chk cos.Chunk) {
  213. defer wg.Done()
  214. fd, err := os.Open(filepath)
  215. assert.Nil(s.T(), err, "Open File Failed")
  216. opt := &cos.ObjectUploadPartOptions{
  217. ContentLength: chk.Size,
  218. }
  219. fd.Seek(chk.OffSet, os.SEEK_SET)
  220. resp, err := s.CClient.Object.UploadPart(context.Background(), name, v.UploadID, chk.Number, io.LimitReader(fd, chk.Size), opt, &cryptoCtx)
  221. assert.Nil(s.T(), err, "UploadPart failed")
  222. mtx.Lock()
  223. optcom.Parts = append(optcom.Parts, cos.Object{
  224. PartNumber: chk.Number, ETag: resp.Header.Get("ETag"),
  225. })
  226. mtx.Unlock()
  227. }(chunk)
  228. }
  229. wg.Wait()
  230. sort.Sort(cos.ObjectList(optcom.Parts))
  231. _, _, err = s.CClient.Object.CompleteMultipartUpload(context.Background(), name, v.UploadID, optcom)
  232. assert.Nil(s.T(), err, "Complete Failed")
  233. // Range解密读取
  234. for i := 0; i < 10; i++ {
  235. math_rand.Seed(time.Now().UnixNano())
  236. rangeStart := math_rand.Int63n(contentLength)
  237. rangeEnd := rangeStart + math_rand.Int63n(contentLength-rangeStart)
  238. if rangeEnd == rangeStart || rangeStart >= contentLength-1 {
  239. continue
  240. }
  241. opt := &cos.ObjectGetOptions{
  242. Range: fmt.Sprintf("bytes=%v-%v", rangeStart, rangeEnd),
  243. Listener: &cos.DefaultProgressListener{},
  244. }
  245. resp, err := s.CClient.Object.Get(context.Background(), name, opt)
  246. assert.Nil(s.T(), err, "GetObject Failed")
  247. defer resp.Body.Close()
  248. decryptedData, _ := ioutil.ReadAll(resp.Body)
  249. assert.Equal(s.T(), bytes.Compare(originData[rangeStart:rangeEnd+1], decryptedData), 0, "decryptData != originData")
  250. }
  251. opt := &cos.ObjectGetOptions{
  252. Listener: &cos.DefaultProgressListener{},
  253. }
  254. resp, err := s.CClient.Object.Get(context.Background(), name, opt)
  255. assert.Nil(s.T(), err, "GetObject Failed")
  256. assert.Equal(s.T(), resp.Header.Get("x-cos-meta-isEncrypted"), "true", "meta data isn't consistent")
  257. assert.Equal(s.T(), resp.Header.Get(coscrypto.COSClientSideEncryptionCekAlg), coscrypto.AesCtrAlgorithm, "meta data isn't consistent")
  258. assert.Equal(s.T(), resp.Header.Get(coscrypto.COSClientSideEncryptionWrapAlg), coscrypto.CosKmsCryptoWrap, "meta data isn't consistent")
  259. assert.Equal(s.T(), resp.Header.Get(coscrypto.COSClientSideEncryptionUnencryptedContentMD5), base64.StdEncoding.EncodeToString(contentMD5), "meta data isn't consistent")
  260. defer resp.Body.Close()
  261. decryptedData, _ := ioutil.ReadAll(resp.Body)
  262. assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData")
  263. _, err = s.CClient.Object.Delete(context.Background(), name)
  264. assert.Nil(s.T(), err, "DeleteObject Failed")
  265. }
  266. func (s *CosTestSuite) TestMultiUpload_GetWithNewClient() {
  267. name := "test/ObjectPut" + time.Now().Format(time.RFC3339)
  268. contentLength := int64(1024*1024*10 + 1)
  269. originData := make([]byte, contentLength)
  270. _, err := rand.Read(originData)
  271. cryptoCtx := coscrypto.CryptoContext{
  272. DataSize: contentLength,
  273. PartSize: (contentLength / 16 / 3) * 16,
  274. }
  275. v, _, err := s.CClient.Object.InitiateMultipartUpload(context.Background(), name, nil, &cryptoCtx)
  276. assert.Nil(s.T(), err, "Init Failed")
  277. chunks, _, err := cos.SplitSizeIntoChunks(contentLength, cryptoCtx.PartSize)
  278. assert.Nil(s.T(), err, "Split Failed")
  279. optcom := &cos.CompleteMultipartUploadOptions{}
  280. for _, chunk := range chunks {
  281. opt := &cos.ObjectUploadPartOptions{
  282. ContentLength: chunk.Size,
  283. }
  284. f := bytes.NewReader(originData[chunk.OffSet : chunk.OffSet+chunk.Size])
  285. resp, err := s.CClient.Object.UploadPart(context.Background(), name, v.UploadID, chunk.Number, io.LimitReader(f, chunk.Size), opt, &cryptoCtx)
  286. assert.Nil(s.T(), err, "UploadPart failed")
  287. optcom.Parts = append(optcom.Parts, cos.Object{
  288. PartNumber: chunk.Number, ETag: resp.Header.Get("ETag"),
  289. })
  290. }
  291. _, _, err = s.CClient.Object.CompleteMultipartUpload(context.Background(), name, v.UploadID, optcom)
  292. assert.Nil(s.T(), err, "Complete Failed")
  293. u, _ := url.Parse("https://" + kBucket + ".cos." + kRegion + ".myqcloud.com")
  294. b := &cos.BaseURL{BucketURL: u}
  295. c := cos.NewClient(b, &http.Client{
  296. Transport: &cos.AuthorizationTransport{
  297. SecretID: os.Getenv("COS_SECRETID"),
  298. SecretKey: os.Getenv("COS_SECRETKEY"),
  299. },
  300. })
  301. {
  302. // 使用不同的MatDesc客户端读取, 期待错误
  303. material := make(map[string]string)
  304. material["desc"] = "cos crypto suite test 2"
  305. kmsclient, _ := coscrypto.NewKMSClient(c.GetCredential(), kRegion)
  306. master, _ := coscrypto.CreateMasterKMS(kmsclient, os.Getenv("KMSID"), material)
  307. client := coscrypto.NewCryptoClient(c, master)
  308. resp, err := client.Object.Get(context.Background(), name, nil)
  309. assert.Nil(s.T(), resp, "Get Object Failed")
  310. assert.NotNil(s.T(), err, "Get Object Failed")
  311. }
  312. {
  313. // 使用相同的MatDesc客户端读取, 但KMSID不一样,期待正确,kms解密是不需要KMSID
  314. material := make(map[string]string)
  315. material["desc"] = "cos crypto suite test"
  316. kmsclient, _ := coscrypto.NewKMSClient(c.GetCredential(), kRegion)
  317. master, _ := coscrypto.CreateMasterKMS(kmsclient, "KMSID", material)
  318. client := coscrypto.NewCryptoClient(c, master)
  319. resp, err := client.Object.Get(context.Background(), name, nil)
  320. assert.Nil(s.T(), err, "Get Object Failed")
  321. defer resp.Body.Close()
  322. decryptedData, _ := ioutil.ReadAll(resp.Body)
  323. assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData")
  324. }
  325. {
  326. // 使用相同的MatDesc客户端读取, 地域不一样,期待错误
  327. material := make(map[string]string)
  328. material["desc"] = "cos crypto suite test"
  329. diffRegion := "ap-shanghai"
  330. if diffRegion == kRegion {
  331. diffRegion = "ap-guangzhou"
  332. }
  333. kmsclient, _ := coscrypto.NewKMSClient(c.GetCredential(), diffRegion)
  334. master, _ := coscrypto.CreateMasterKMS(kmsclient, "KMSID", material)
  335. client := coscrypto.NewCryptoClient(c, master)
  336. resp, err := client.Object.Get(context.Background(), name, nil)
  337. assert.Nil(s.T(), resp, "Get Object Failed")
  338. assert.NotNil(s.T(), err, "Get Object Failed")
  339. }
  340. {
  341. // 使用相同的MatDesc和KMSID客户端读取, 期待正确
  342. material := make(map[string]string)
  343. material["desc"] = "cos crypto suite test"
  344. kmsclient, _ := coscrypto.NewKMSClient(c.GetCredential(), kRegion)
  345. master, _ := coscrypto.CreateMasterKMS(kmsclient, os.Getenv("KMSID"), material)
  346. client := coscrypto.NewCryptoClient(c, master)
  347. resp, err := client.Object.Get(context.Background(), name, nil)
  348. assert.Nil(s.T(), err, "Get Object Failed")
  349. defer resp.Body.Close()
  350. decryptedData, _ := ioutil.ReadAll(resp.Body)
  351. assert.Equal(s.T(), bytes.Compare(originData, decryptedData), 0, "decryptData != originData")
  352. }
  353. _, err = s.CClient.Object.Delete(context.Background(), name)
  354. assert.Nil(s.T(), err, "DeleteObject Failed")
  355. }