You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

1374 lines
42 KiB

4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
  1. package cos
  2. import (
  3. "context"
  4. "crypto/md5"
  5. "encoding/json"
  6. "encoding/xml"
  7. "errors"
  8. "fmt"
  9. "hash/crc64"
  10. "io"
  11. "io/ioutil"
  12. "net/http"
  13. "net/url"
  14. "os"
  15. "sort"
  16. "strconv"
  17. "strings"
  18. "time"
  19. )
  20. // ObjectService 相关 API
  21. type ObjectService service
  22. // ObjectGetOptions is the option of GetObject
  23. type ObjectGetOptions struct {
  24. ResponseContentType string `url:"response-content-type,omitempty" header:"-"`
  25. ResponseContentLanguage string `url:"response-content-language,omitempty" header:"-"`
  26. ResponseExpires string `url:"response-expires,omitempty" header:"-"`
  27. ResponseCacheControl string `url:"response-cache-control,omitempty" header:"-"`
  28. ResponseContentDisposition string `url:"response-content-disposition,omitempty" header:"-"`
  29. ResponseContentEncoding string `url:"response-content-encoding,omitempty" header:"-"`
  30. Range string `url:"-" header:"Range,omitempty"`
  31. IfModifiedSince string `url:"-" header:"If-Modified-Since,omitempty"`
  32. // SSE-C
  33. XCosSSECustomerAglo string `header:"x-cos-server-side-encryption-customer-algorithm,omitempty" url:"-" xml:"-"`
  34. XCosSSECustomerKey string `header:"x-cos-server-side-encryption-customer-key,omitempty" url:"-" xml:"-"`
  35. XCosSSECustomerKeyMD5 string `header:"x-cos-server-side-encryption-customer-key-MD5,omitempty" url:"-" xml:"-"`
  36. //兼容其他自定义头部
  37. XOptionHeader *http.Header `header:"-,omitempty" url:"-" xml:"-"`
  38. XCosTrafficLimit int `header:"x-cos-traffic-limit,omitempty" url:"-" xml:"-"`
  39. // 下载进度, ProgressCompleteEvent不能表示对应API调用成功,API是否调用成功的判断标准为返回err==nil
  40. Listener ProgressListener `header:"-" url:"-" xml:"-"`
  41. }
  42. // presignedURLTestingOptions is the opt of presigned url
  43. type presignedURLTestingOptions struct {
  44. authTime *AuthTime
  45. }
  46. // Get Object 请求可以将一个文件(Object)下载至本地。
  47. // 该操作需要对目标 Object 具有读权限或目标 Object 对所有人都开放了读权限(公有读)。
  48. //
  49. // https://www.qcloud.com/document/product/436/7753
  50. func (s *ObjectService) Get(ctx context.Context, name string, opt *ObjectGetOptions, id ...string) (*Response, error) {
  51. var u string
  52. if len(id) == 1 {
  53. u = fmt.Sprintf("/%s?versionId=%s", encodeURIComponent(name), id[0])
  54. } else if len(id) == 0 {
  55. u = "/" + encodeURIComponent(name)
  56. } else {
  57. return nil, errors.New("wrong params")
  58. }
  59. sendOpt := sendOptions{
  60. baseURL: s.client.BaseURL.BucketURL,
  61. uri: u,
  62. method: http.MethodGet,
  63. optQuery: opt,
  64. optHeader: opt,
  65. disableCloseBody: true,
  66. }
  67. resp, err := s.client.send(ctx, &sendOpt)
  68. if opt != nil && opt.Listener != nil {
  69. if err == nil && resp != nil {
  70. if totalBytes, e := strconv.ParseInt(resp.Header.Get("Content-Length"), 10, 64); e == nil {
  71. resp.Body = TeeReader(resp.Body, nil, totalBytes, opt.Listener)
  72. }
  73. }
  74. }
  75. return resp, err
  76. }
  77. // GetToFile download the object to local file
  78. func (s *ObjectService) GetToFile(ctx context.Context, name, localpath string, opt *ObjectGetOptions, id ...string) (*Response, error) {
  79. resp, err := s.Get(ctx, name, opt, id...)
  80. if err != nil {
  81. return resp, err
  82. }
  83. defer resp.Body.Close()
  84. // If file exist, overwrite it
  85. fd, err := os.OpenFile(localpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660)
  86. if err != nil {
  87. return resp, err
  88. }
  89. _, err = io.Copy(fd, resp.Body)
  90. fd.Close()
  91. if err != nil {
  92. return resp, err
  93. }
  94. return resp, nil
  95. }
  96. type PresignedURLOptions struct {
  97. Query *url.Values `xml:"-" url:"-" header:"-"`
  98. Header *http.Header `header:"-,omitempty" url:"-" xml:"-"`
  99. }
  100. // GetPresignedURL get the object presigned to down or upload file by url
  101. func (s *ObjectService) GetPresignedURL(ctx context.Context, httpMethod, name, ak, sk string, expired time.Duration, opt interface{}) (*url.URL, error) {
  102. sendOpt := sendOptions{
  103. baseURL: s.client.BaseURL.BucketURL,
  104. uri: "/" + encodeURIComponent(name),
  105. method: httpMethod,
  106. optQuery: opt,
  107. optHeader: opt,
  108. }
  109. if popt, ok := opt.(*PresignedURLOptions); ok {
  110. qs := popt.Query.Encode()
  111. if qs != "" {
  112. sendOpt.uri = fmt.Sprintf("%s?%s", sendOpt.uri, qs)
  113. }
  114. }
  115. req, err := s.client.newRequest(ctx, sendOpt.baseURL, sendOpt.uri, sendOpt.method, sendOpt.body, sendOpt.optQuery, sendOpt.optHeader)
  116. if err != nil {
  117. return nil, err
  118. }
  119. var authTime *AuthTime
  120. if opt != nil {
  121. if opt, ok := opt.(*presignedURLTestingOptions); ok {
  122. authTime = opt.authTime
  123. }
  124. }
  125. if authTime == nil {
  126. authTime = NewAuthTime(expired)
  127. }
  128. authorization := newAuthorization(ak, sk, req, authTime)
  129. sign := encodeURIComponent(authorization, []byte{'&', '='})
  130. if req.URL.RawQuery == "" {
  131. req.URL.RawQuery = fmt.Sprintf("%s", sign)
  132. } else {
  133. req.URL.RawQuery = fmt.Sprintf("%s&%s", req.URL.RawQuery, sign)
  134. }
  135. return req.URL, nil
  136. }
  137. // ObjectPutHeaderOptions the options of header of the put object
  138. type ObjectPutHeaderOptions struct {
  139. CacheControl string `header:"Cache-Control,omitempty" url:"-"`
  140. ContentDisposition string `header:"Content-Disposition,omitempty" url:"-"`
  141. ContentEncoding string `header:"Content-Encoding,omitempty" url:"-"`
  142. ContentType string `header:"Content-Type,omitempty" url:"-"`
  143. ContentMD5 string `header:"Content-MD5,omitempty" url:"-"`
  144. ContentLength int64 `header:"Content-Length,omitempty" url:"-"`
  145. ContentLanguage string `header:"Content-Language,omitempty" url:"-"`
  146. Expect string `header:"Expect,omitempty" url:"-"`
  147. Expires string `header:"Expires,omitempty" url:"-"`
  148. XCosContentSHA1 string `header:"x-cos-content-sha1,omitempty" url:"-"`
  149. // 自定义的 x-cos-meta-* header
  150. XCosMetaXXX *http.Header `header:"x-cos-meta-*,omitempty" url:"-"`
  151. XCosStorageClass string `header:"x-cos-storage-class,omitempty" url:"-"`
  152. // 可选值: Normal, Appendable
  153. //XCosObjectType string `header:"x-cos-object-type,omitempty" url:"-"`
  154. // Enable Server Side Encryption, Only supported: AES256
  155. XCosServerSideEncryption string `header:"x-cos-server-side-encryption,omitempty" url:"-" xml:"-"`
  156. // SSE-C
  157. XCosSSECustomerAglo string `header:"x-cos-server-side-encryption-customer-algorithm,omitempty" url:"-" xml:"-"`
  158. XCosSSECustomerKey string `header:"x-cos-server-side-encryption-customer-key,omitempty" url:"-" xml:"-"`
  159. XCosSSECustomerKeyMD5 string `header:"x-cos-server-side-encryption-customer-key-MD5,omitempty" url:"-" xml:"-"`
  160. //兼容其他自定义头部
  161. XOptionHeader *http.Header `header:"-,omitempty" url:"-" xml:"-"`
  162. XCosTrafficLimit int `header:"x-cos-traffic-limit,omitempty" url:"-" xml:"-"`
  163. // 上传进度, ProgressCompleteEvent不能表示对应API调用成功,API是否调用成功的判断标准为返回err==nil
  164. Listener ProgressListener `header:"-" url:"-" xml:"-"`
  165. }
  166. // ObjectPutOptions the options of put object
  167. type ObjectPutOptions struct {
  168. *ACLHeaderOptions `header:",omitempty" url:"-" xml:"-"`
  169. *ObjectPutHeaderOptions `header:",omitempty" url:"-" xml:"-"`
  170. }
  171. // Put Object请求可以将一个文件(Oject)上传至指定Bucket。
  172. //
  173. // https://www.qcloud.com/document/product/436/7749
  174. func (s *ObjectService) Put(ctx context.Context, name string, r io.Reader, uopt *ObjectPutOptions) (*Response, error) {
  175. if r == nil {
  176. return nil, fmt.Errorf("reader is nil")
  177. }
  178. if err := CheckReaderLen(r); err != nil {
  179. return nil, err
  180. }
  181. opt := CloneObjectPutOptions(uopt)
  182. totalBytes, err := GetReaderLen(r)
  183. if err != nil && opt != nil && opt.Listener != nil {
  184. if opt.ContentLength == 0 {
  185. return nil, err
  186. }
  187. totalBytes = opt.ContentLength
  188. }
  189. if err == nil {
  190. // 与 go http 保持一致, 非bytes.Buffer/bytes.Reader/strings.Reader由用户指定ContentLength, 或使用 Chunk 上传
  191. if opt != nil && opt.ContentLength == 0 && IsLenReader(r) {
  192. opt.ContentLength = totalBytes
  193. }
  194. }
  195. reader := TeeReader(r, nil, totalBytes, nil)
  196. if s.client.Conf.EnableCRC {
  197. reader.writer = crc64.New(crc64.MakeTable(crc64.ECMA))
  198. }
  199. if opt != nil && opt.Listener != nil {
  200. reader.listener = opt.Listener
  201. }
  202. sendOpt := sendOptions{
  203. baseURL: s.client.BaseURL.BucketURL,
  204. uri: "/" + encodeURIComponent(name),
  205. method: http.MethodPut,
  206. body: reader,
  207. optHeader: opt,
  208. }
  209. resp, err := s.client.send(ctx, &sendOpt)
  210. return resp, err
  211. }
  212. // PutFromFile put object from local file
  213. // Notice that when use this put large file need set non-body of debug req/resp, otherwise will out of memory
  214. func (s *ObjectService) PutFromFile(ctx context.Context, name string, filePath string, opt *ObjectPutOptions) (resp *Response, err error) {
  215. nr := 0
  216. for nr < 3 {
  217. fd, e := os.Open(filePath)
  218. if e != nil {
  219. err = e
  220. return
  221. }
  222. resp, err = s.Put(ctx, name, fd, opt)
  223. if err != nil {
  224. nr++
  225. fd.Close()
  226. continue
  227. }
  228. fd.Close()
  229. break
  230. }
  231. return
  232. }
  233. // ObjectCopyHeaderOptions is the head option of the Copy
  234. type ObjectCopyHeaderOptions struct {
  235. // When use replace directive to update meta infos
  236. CacheControl string `header:"Cache-Control,omitempty" url:"-"`
  237. ContentDisposition string `header:"Content-Disposition,omitempty" url:"-"`
  238. ContentEncoding string `header:"Content-Encoding,omitempty" url:"-"`
  239. ContentLanguage string `header:"Content-Language,omitempty" url:"-"`
  240. ContentType string `header:"Content-Type,omitempty" url:"-"`
  241. Expires string `header:"Expires,omitempty" url:"-"`
  242. Expect string `header:"Expect,omitempty" url:"-"`
  243. XCosMetadataDirective string `header:"x-cos-metadata-directive,omitempty" url:"-" xml:"-"`
  244. XCosCopySourceIfModifiedSince string `header:"x-cos-copy-source-If-Modified-Since,omitempty" url:"-" xml:"-"`
  245. XCosCopySourceIfUnmodifiedSince string `header:"x-cos-copy-source-If-Unmodified-Since,omitempty" url:"-" xml:"-"`
  246. XCosCopySourceIfMatch string `header:"x-cos-copy-source-If-Match,omitempty" url:"-" xml:"-"`
  247. XCosCopySourceIfNoneMatch string `header:"x-cos-copy-source-If-None-Match,omitempty" url:"-" xml:"-"`
  248. XCosStorageClass string `header:"x-cos-storage-class,omitempty" url:"-" xml:"-"`
  249. // 自定义的 x-cos-meta-* header
  250. XCosMetaXXX *http.Header `header:"x-cos-meta-*,omitempty" url:"-"`
  251. XCosCopySource string `header:"x-cos-copy-source" url:"-" xml:"-"`
  252. XCosServerSideEncryption string `header:"x-cos-server-side-encryption,omitempty" url:"-" xml:"-"`
  253. // SSE-C
  254. XCosSSECustomerAglo string `header:"x-cos-server-side-encryption-customer-algorithm,omitempty" url:"-" xml:"-"`
  255. XCosSSECustomerKey string `header:"x-cos-server-side-encryption-customer-key,omitempty" url:"-" xml:"-"`
  256. XCosSSECustomerKeyMD5 string `header:"x-cos-server-side-encryption-customer-key-MD5,omitempty" url:"-" xml:"-"`
  257. XCosCopySourceSSECustomerAglo string `header:"x-cos-copy-source-server-side-encryption-customer-algorithm,omitempty" url:"-" xml:"-"`
  258. XCosCopySourceSSECustomerKey string `header:"x-cos-copy-source-server-side-encryption-customer-key,omitempty" url:"-" xml:"-"`
  259. XCosCopySourceSSECustomerKeyMD5 string `header:"x-cos-copy-source-server-side-encryption-customer-key-MD5,omitempty" url:"-" xml:"-"`
  260. //兼容其他自定义头部
  261. XOptionHeader *http.Header `header:"-,omitempty" url:"-" xml:"-"`
  262. }
  263. // ObjectCopyOptions is the option of Copy, choose header or body
  264. type ObjectCopyOptions struct {
  265. *ObjectCopyHeaderOptions `header:",omitempty" url:"-" xml:"-"`
  266. *ACLHeaderOptions `header:",omitempty" url:"-" xml:"-"`
  267. }
  268. // ObjectCopyResult is the result of Copy
  269. type ObjectCopyResult struct {
  270. XMLName xml.Name `xml:"CopyObjectResult"`
  271. ETag string `xml:"ETag,omitempty"`
  272. LastModified string `xml:"LastModified,omitempty"`
  273. CRC64 string `xml:"CRC64,omitempty"`
  274. VersionId string `xml:"VersionId,omitempty"`
  275. }
  276. // Copy 调用 PutObjectCopy 请求实现将一个文件从源路径复制到目标路径。建议文件大小 1M 到 5G,
  277. // 超过 5G 的文件请使用分块上传 Upload - Copy。在拷贝的过程中,文件元属性和 ACL 可以被修改。
  278. //
  279. // 用户可以通过该接口实现文件移动,文件重命名,修改文件属性和创建副本。
  280. //
  281. // 注意:在跨帐号复制的时候,需要先设置被复制文件的权限为公有读,或者对目标帐号赋权,同帐号则不需要。
  282. //
  283. // https://cloud.tencent.com/document/product/436/10881
  284. func (s *ObjectService) Copy(ctx context.Context, name, sourceURL string, opt *ObjectCopyOptions, id ...string) (*ObjectCopyResult, *Response, error) {
  285. surl := strings.SplitN(sourceURL, "/", 2)
  286. if len(surl) < 2 {
  287. return nil, nil, errors.New(fmt.Sprintf("x-cos-copy-source format error: %s", sourceURL))
  288. }
  289. var u string
  290. if len(id) == 1 {
  291. u = fmt.Sprintf("%s/%s?versionId=%s", surl[0], encodeURIComponent(surl[1]), id[0])
  292. } else if len(id) == 0 {
  293. u = fmt.Sprintf("%s/%s", surl[0], encodeURIComponent(surl[1]))
  294. } else {
  295. return nil, nil, errors.New("wrong params")
  296. }
  297. var res ObjectCopyResult
  298. copyOpt := &ObjectCopyOptions{
  299. &ObjectCopyHeaderOptions{},
  300. &ACLHeaderOptions{},
  301. }
  302. if opt != nil {
  303. if opt.ObjectCopyHeaderOptions != nil {
  304. *copyOpt.ObjectCopyHeaderOptions = *opt.ObjectCopyHeaderOptions
  305. }
  306. if opt.ACLHeaderOptions != nil {
  307. *copyOpt.ACLHeaderOptions = *opt.ACLHeaderOptions
  308. }
  309. }
  310. copyOpt.XCosCopySource = u
  311. sendOpt := sendOptions{
  312. baseURL: s.client.BaseURL.BucketURL,
  313. uri: "/" + encodeURIComponent(name),
  314. method: http.MethodPut,
  315. body: nil,
  316. optHeader: copyOpt,
  317. result: &res,
  318. }
  319. resp, err := s.client.send(ctx, &sendOpt)
  320. // If the error occurs during the copy operation, the error response is embedded in the 200 OK response. This means that a 200 OK response can contain either a success or an error.
  321. if err == nil && resp.StatusCode == 200 {
  322. if res.ETag == "" {
  323. return &res, resp, errors.New("response 200 OK, but body contains an error")
  324. }
  325. }
  326. return &res, resp, err
  327. }
  328. type ObjectDeleteOptions struct {
  329. // SSE-C
  330. XCosSSECustomerAglo string `header:"x-cos-server-side-encryption-customer-algorithm,omitempty" url:"-" xml:"-"`
  331. XCosSSECustomerKey string `header:"x-cos-server-side-encryption-customer-key,omitempty" url:"-" xml:"-"`
  332. XCosSSECustomerKeyMD5 string `header:"x-cos-server-side-encryption-customer-key-MD5,omitempty" url:"-" xml:"-"`
  333. //兼容其他自定义头部
  334. XOptionHeader *http.Header `header:"-,omitempty" url:"-" xml:"-"`
  335. VersionId string `header:"-" url:"VersionId,omitempty" xml:"-"`
  336. }
  337. // Delete Object请求可以将一个文件(Object)删除。
  338. //
  339. // https://www.qcloud.com/document/product/436/7743
  340. func (s *ObjectService) Delete(ctx context.Context, name string, opt ...*ObjectDeleteOptions) (*Response, error) {
  341. var optHeader *ObjectDeleteOptions
  342. // When use "" string might call the delete bucket interface
  343. if len(name) == 0 {
  344. return nil, errors.New("empty object name")
  345. }
  346. if len(opt) > 0 {
  347. optHeader = opt[0]
  348. }
  349. sendOpt := sendOptions{
  350. baseURL: s.client.BaseURL.BucketURL,
  351. uri: "/" + encodeURIComponent(name),
  352. method: http.MethodDelete,
  353. optHeader: optHeader,
  354. optQuery: optHeader,
  355. }
  356. resp, err := s.client.send(ctx, &sendOpt)
  357. return resp, err
  358. }
  359. // ObjectHeadOptions is the option of HeadObject
  360. type ObjectHeadOptions struct {
  361. IfModifiedSince string `url:"-" header:"If-Modified-Since,omitempty"`
  362. // SSE-C
  363. XCosSSECustomerAglo string `header:"x-cos-server-side-encryption-customer-algorithm,omitempty" url:"-" xml:"-"`
  364. XCosSSECustomerKey string `header:"x-cos-server-side-encryption-customer-key,omitempty" url:"-" xml:"-"`
  365. XCosSSECustomerKeyMD5 string `header:"x-cos-server-side-encryption-customer-key-MD5,omitempty" url:"-" xml:"-"`
  366. XOptionHeader *http.Header `header:"-,omitempty" url:"-" xml:"-"`
  367. }
  368. // Head Object请求可以取回对应Object的元数据,Head的权限与Get的权限一致
  369. //
  370. // https://www.qcloud.com/document/product/436/7745
  371. func (s *ObjectService) Head(ctx context.Context, name string, opt *ObjectHeadOptions, id ...string) (*Response, error) {
  372. var u string
  373. if len(id) == 1 {
  374. u = fmt.Sprintf("/%s?versionId=%s", encodeURIComponent(name), id[0])
  375. } else if len(id) == 0 {
  376. u = "/" + encodeURIComponent(name)
  377. } else {
  378. return nil, errors.New("wrong params")
  379. }
  380. sendOpt := sendOptions{
  381. baseURL: s.client.BaseURL.BucketURL,
  382. uri: u,
  383. method: http.MethodHead,
  384. optHeader: opt,
  385. }
  386. resp, err := s.client.send(ctx, &sendOpt)
  387. if resp != nil && resp.Header["X-Cos-Object-Type"] != nil && resp.Header["X-Cos-Object-Type"][0] == "appendable" {
  388. resp.Header.Add("x-cos-next-append-position", resp.Header["Content-Length"][0])
  389. }
  390. return resp, err
  391. }
  392. // ObjectOptionsOptions is the option of object options
  393. type ObjectOptionsOptions struct {
  394. Origin string `url:"-" header:"Origin"`
  395. AccessControlRequestMethod string `url:"-" header:"Access-Control-Request-Method"`
  396. AccessControlRequestHeaders string `url:"-" header:"Access-Control-Request-Headers,omitempty"`
  397. }
  398. // Options Object请求实现跨域访问的预请求。即发出一个 OPTIONS 请求给服务器以确认是否可以进行跨域操作。
  399. //
  400. // 当CORS配置不存在时,请求返回403 Forbidden。
  401. //
  402. // https://www.qcloud.com/document/product/436/8288
  403. func (s *ObjectService) Options(ctx context.Context, name string, opt *ObjectOptionsOptions) (*Response, error) {
  404. sendOpt := sendOptions{
  405. baseURL: s.client.BaseURL.BucketURL,
  406. uri: "/" + encodeURIComponent(name),
  407. method: http.MethodOptions,
  408. optHeader: opt,
  409. }
  410. resp, err := s.client.send(ctx, &sendOpt)
  411. return resp, err
  412. }
  413. // CASJobParameters support three way: Standard(in 35 hours), Expedited(quick way, in 15 mins), Bulk(in 5-12 hours_
  414. type CASJobParameters struct {
  415. Tier string `xml:"Tier"`
  416. }
  417. // ObjectRestoreOptions is the option of object restore
  418. type ObjectRestoreOptions struct {
  419. XMLName xml.Name `xml:"RestoreRequest"`
  420. Days int `xml:"Days"`
  421. Tier *CASJobParameters `xml:"CASJobParameters"`
  422. }
  423. // PutRestore API can recover an object of type archived by COS archive.
  424. //
  425. // https://cloud.tencent.com/document/product/436/12633
  426. func (s *ObjectService) PostRestore(ctx context.Context, name string, opt *ObjectRestoreOptions) (*Response, error) {
  427. u := fmt.Sprintf("/%s?restore", encodeURIComponent(name))
  428. sendOpt := sendOptions{
  429. baseURL: s.client.BaseURL.BucketURL,
  430. uri: u,
  431. method: http.MethodPost,
  432. body: opt,
  433. }
  434. resp, err := s.client.send(ctx, &sendOpt)
  435. return resp, err
  436. }
  437. // TODO Append 接口在优化未开放使用
  438. //
  439. // Append请求可以将一个文件(Object)以分块追加的方式上传至 Bucket 中。使用Append Upload的文件必须事前被设定为Appendable。
  440. // 当Appendable的文件被执行Put Object的操作以后,文件被覆盖,属性改变为Normal。
  441. //
  442. // 文件属性可以在Head Object操作中被查询到,当您发起Head Object请求时,会返回自定义Header『x-cos-object-type』,该Header只有两个枚举值:Normal或者Appendable。
  443. //
  444. // 追加上传建议文件大小1M - 5G。如果position的值和当前Object的长度不致,COS会返回409错误。
  445. // 如果Append一个Normal的Object,COS会返回409 ObjectNotAppendable。
  446. //
  447. // Appendable的文件不可以被复制,不参与版本管理,不参与生命周期管理,不可跨区域复制。
  448. //
  449. // 当 r 不是 bytes.Buffer/bytes.Reader/strings.Reader 时,必须指定 opt.ObjectPutHeaderOptions.ContentLength
  450. //
  451. // https://www.qcloud.com/document/product/436/7741
  452. // func (s *ObjectService) Append(ctx context.Context, name string, position int, r io.Reader, opt *ObjectPutOptions) (*Response, error) {
  453. // u := fmt.Sprintf("/%s?append&position=%d", encodeURIComponent(name), position)
  454. // if position != 0{
  455. // opt = nil
  456. // }
  457. // sendOpt := sendOptions{
  458. // baseURL: s.client.BaseURL.BucketURL,
  459. // uri: u,
  460. // method: http.MethodPost,
  461. // optHeader: opt,
  462. // body: r,
  463. // }
  464. // resp, err := s.client.send(ctx, &sendOpt)
  465. // return resp, err
  466. // }
  467. // ObjectDeleteMultiOptions is the option of DeleteMulti
  468. type ObjectDeleteMultiOptions struct {
  469. XMLName xml.Name `xml:"Delete" header:"-"`
  470. Quiet bool `xml:"Quiet" header:"-"`
  471. Objects []Object `xml:"Object" header:"-"`
  472. //XCosSha1 string `xml:"-" header:"x-cos-sha1"`
  473. }
  474. // ObjectDeleteMultiResult is the result of DeleteMulti
  475. type ObjectDeleteMultiResult struct {
  476. XMLName xml.Name `xml:"DeleteResult"`
  477. DeletedObjects []Object `xml:"Deleted,omitempty"`
  478. Errors []struct {
  479. Key string `xml:",omitempty"`
  480. Code string `xml:",omitempty"`
  481. Message string `xml:",omitempty"`
  482. VersionId string `xml:",omitempty"`
  483. } `xml:"Error,omitempty"`
  484. }
  485. // DeleteMulti 请求实现批量删除文件,最大支持单次删除1000个文件。
  486. // 对于返回结果,COS提供Verbose和Quiet两种结果模式。Verbose模式将返回每个Object的删除结果;
  487. // Quiet模式只返回报错的Object信息。
  488. // https://www.qcloud.com/document/product/436/8289
  489. func (s *ObjectService) DeleteMulti(ctx context.Context, opt *ObjectDeleteMultiOptions) (*ObjectDeleteMultiResult, *Response, error) {
  490. var res ObjectDeleteMultiResult
  491. sendOpt := sendOptions{
  492. baseURL: s.client.BaseURL.BucketURL,
  493. uri: "/?delete",
  494. method: http.MethodPost,
  495. body: opt,
  496. result: &res,
  497. }
  498. resp, err := s.client.send(ctx, &sendOpt)
  499. return &res, resp, err
  500. }
  501. // Object is the meta info of the object
  502. type Object struct {
  503. Key string `xml:",omitempty"`
  504. ETag string `xml:",omitempty"`
  505. Size int64 `xml:",omitempty"`
  506. PartNumber int `xml:",omitempty"`
  507. LastModified string `xml:",omitempty"`
  508. StorageClass string `xml:",omitempty"`
  509. Owner *Owner `xml:",omitempty"`
  510. VersionId string `xml:",omitempty"`
  511. }
  512. // MultiUploadOptions is the option of the multiupload,
  513. // ThreadPoolSize default is one
  514. type MultiUploadOptions struct {
  515. OptIni *InitiateMultipartUploadOptions
  516. PartSize int64
  517. ThreadPoolSize int
  518. CheckPoint bool
  519. EnableVerification bool
  520. }
  521. type MultiDownloadOptions struct {
  522. Opt *ObjectGetOptions
  523. PartSize int64
  524. ThreadPoolSize int
  525. CheckPoint bool
  526. CheckPointFile string
  527. }
  528. type MultiDownloadCPInfo struct {
  529. Size int64 `json:"contentLength,omitempty"`
  530. ETag string `json:"eTag,omitempty"`
  531. CRC64 string `json:"crc64ecma,omitempty"`
  532. LastModified string `json:"lastModified,omitempty"`
  533. DownloadedBlocks []DownloadedBlock `json:"downloadedBlocks,omitempty"`
  534. }
  535. type DownloadedBlock struct {
  536. From int64 `json:"from,omitempty"`
  537. To int64 `json:"to,omitempty"`
  538. }
  539. type Chunk struct {
  540. Number int
  541. OffSet int64
  542. Size int64
  543. Done bool
  544. ETag string
  545. }
  546. // jobs
  547. type Jobs struct {
  548. Name string
  549. UploadId string
  550. FilePath string
  551. RetryTimes int
  552. VersionId []string
  553. Chunk Chunk
  554. Data io.Reader
  555. Opt *ObjectUploadPartOptions
  556. DownOpt *ObjectGetOptions
  557. }
  558. type Results struct {
  559. PartNumber int
  560. Resp *Response
  561. err error
  562. }
  563. func LimitReadCloser(r io.Reader, n int64) io.Reader {
  564. var lc LimitedReadCloser
  565. lc.R = r
  566. lc.N = n
  567. return &lc
  568. }
  569. type LimitedReadCloser struct {
  570. io.LimitedReader
  571. }
  572. func (lc *LimitedReadCloser) Close() error {
  573. if r, ok := lc.R.(io.ReadCloser); ok {
  574. return r.Close()
  575. }
  576. return nil
  577. }
  578. type DiscardReadCloser struct {
  579. RC io.ReadCloser
  580. Discard int
  581. }
  582. func (drc *DiscardReadCloser) Read(data []byte) (int, error) {
  583. n, err := drc.RC.Read(data)
  584. if drc.Discard == 0 || n <= 0 {
  585. return n, err
  586. }
  587. if n <= drc.Discard {
  588. drc.Discard -= n
  589. return 0, err
  590. }
  591. realLen := n - drc.Discard
  592. copy(data[0:realLen], data[drc.Discard:n])
  593. drc.Discard = 0
  594. return realLen, err
  595. }
  596. func (drc *DiscardReadCloser) Close() error {
  597. if rc, ok := drc.RC.(io.ReadCloser); ok {
  598. return rc.Close()
  599. }
  600. return nil
  601. }
  602. func worker(s *ObjectService, jobs <-chan *Jobs, results chan<- *Results) {
  603. for j := range jobs {
  604. j.Opt.ContentLength = j.Chunk.Size
  605. rt := j.RetryTimes
  606. for {
  607. // http.Request.Body can be Closed in request
  608. fd, err := os.Open(j.FilePath)
  609. var res Results
  610. if err != nil {
  611. res.err = err
  612. res.PartNumber = j.Chunk.Number
  613. res.Resp = nil
  614. results <- &res
  615. break
  616. }
  617. fd.Seek(j.Chunk.OffSet, os.SEEK_SET)
  618. resp, err := s.UploadPart(context.Background(), j.Name, j.UploadId, j.Chunk.Number,
  619. LimitReadCloser(fd, j.Chunk.Size), j.Opt)
  620. res.PartNumber = j.Chunk.Number
  621. res.Resp = resp
  622. res.err = err
  623. if err != nil {
  624. rt--
  625. if rt == 0 {
  626. results <- &res
  627. break
  628. }
  629. continue
  630. }
  631. results <- &res
  632. break
  633. }
  634. }
  635. }
  636. func downloadWorker(s *ObjectService, jobs <-chan *Jobs, results chan<- *Results) {
  637. for j := range jobs {
  638. opt := &RangeOptions{
  639. HasStart: true,
  640. HasEnd: true,
  641. Start: j.Chunk.OffSet,
  642. End: j.Chunk.OffSet + j.Chunk.Size - 1,
  643. }
  644. j.DownOpt.Range = FormatRangeOptions(opt)
  645. rt := j.RetryTimes
  646. for {
  647. var res Results
  648. res.PartNumber = j.Chunk.Number
  649. resp, err := s.Get(context.Background(), j.Name, j.DownOpt, j.VersionId...)
  650. res.err = err
  651. res.Resp = resp
  652. if err != nil {
  653. rt--
  654. if rt == 0 {
  655. results <- &res
  656. break
  657. }
  658. continue
  659. }
  660. defer resp.Body.Close()
  661. fd, err := os.OpenFile(j.FilePath, os.O_WRONLY, 0660)
  662. if err != nil {
  663. res.err = err
  664. results <- &res
  665. break
  666. }
  667. fd.Seek(j.Chunk.OffSet, os.SEEK_SET)
  668. n, err := io.Copy(fd, LimitReadCloser(resp.Body, j.Chunk.Size))
  669. if n != j.Chunk.Size || err != nil {
  670. res.err = fmt.Errorf("io.Copy Failed, nread:%v, want:%v, err:%v", n, j.Chunk.Size, err)
  671. }
  672. fd.Close()
  673. results <- &res
  674. break
  675. }
  676. }
  677. }
  678. func DividePart(fileSize int64, last int) (int64, int64) {
  679. partSize := int64(last * 1024 * 1024)
  680. partNum := fileSize / partSize
  681. for partNum >= 10000 {
  682. partSize = partSize * 2
  683. partNum = fileSize / partSize
  684. }
  685. return partNum, partSize
  686. }
  687. func SplitFileIntoChunks(filePath string, partSize int64) (int64, []Chunk, int, error) {
  688. if filePath == "" {
  689. return 0, nil, 0, errors.New("filePath invalid")
  690. }
  691. file, err := os.Open(filePath)
  692. if err != nil {
  693. return 0, nil, 0, err
  694. }
  695. defer file.Close()
  696. stat, err := file.Stat()
  697. if err != nil {
  698. return 0, nil, 0, err
  699. }
  700. var partNum int64
  701. if partSize > 0 {
  702. partNum = stat.Size() / partSize
  703. if partNum >= 10000 {
  704. return 0, nil, 0, errors.New("Too many parts, out of 10000")
  705. }
  706. } else {
  707. partNum, partSize = DividePart(stat.Size(), 64)
  708. }
  709. var chunks []Chunk
  710. var chunk = Chunk{}
  711. for i := int64(0); i < partNum; i++ {
  712. chunk.Number = int(i + 1)
  713. chunk.OffSet = i * partSize
  714. chunk.Size = partSize
  715. chunks = append(chunks, chunk)
  716. }
  717. if stat.Size()%partSize > 0 {
  718. chunk.Number = len(chunks) + 1
  719. chunk.OffSet = int64(len(chunks)) * partSize
  720. chunk.Size = stat.Size() % partSize
  721. chunks = append(chunks, chunk)
  722. partNum++
  723. }
  724. return int64(stat.Size()), chunks, int(partNum), nil
  725. }
  726. func (s *ObjectService) getResumableUploadID(ctx context.Context, name string) (string, error) {
  727. opt := &ObjectListUploadsOptions{
  728. Prefix: name,
  729. EncodingType: "url",
  730. }
  731. res, _, err := s.ListUploads(ctx, opt)
  732. if err != nil {
  733. return "", err
  734. }
  735. if len(res.Upload) == 0 {
  736. return "", nil
  737. }
  738. last := len(res.Upload) - 1
  739. for last >= 0 {
  740. decodeKey, _ := decodeURIComponent(res.Upload[last].Key)
  741. if decodeKey == name {
  742. return decodeURIComponent(res.Upload[last].UploadID)
  743. }
  744. last = last - 1
  745. }
  746. return "", nil
  747. }
  748. func (s *ObjectService) checkUploadedParts(ctx context.Context, name, UploadID, filepath string, chunks []Chunk, partNum int) error {
  749. var uploadedParts []Object
  750. isTruncated := true
  751. opt := &ObjectListPartsOptions{
  752. EncodingType: "url",
  753. }
  754. for isTruncated {
  755. res, _, err := s.ListParts(ctx, name, UploadID, opt)
  756. if err != nil {
  757. return err
  758. }
  759. if len(res.Parts) > 0 {
  760. uploadedParts = append(uploadedParts, res.Parts...)
  761. }
  762. isTruncated = res.IsTruncated
  763. opt.PartNumberMarker = res.NextPartNumberMarker
  764. }
  765. fd, err := os.Open(filepath)
  766. if err != nil {
  767. return err
  768. }
  769. defer fd.Close()
  770. // 某个分块出错, 重置chunks
  771. ret := func(e error) error {
  772. for i, _ := range chunks {
  773. chunks[i].Done = false
  774. chunks[i].ETag = ""
  775. }
  776. return e
  777. }
  778. for _, part := range uploadedParts {
  779. partNumber := part.PartNumber
  780. if partNumber > partNum {
  781. return ret(errors.New("Part Number is not consistent"))
  782. }
  783. partNumber = partNumber - 1
  784. fd.Seek(chunks[partNumber].OffSet, os.SEEK_SET)
  785. bs, err := ioutil.ReadAll(io.LimitReader(fd, chunks[partNumber].Size))
  786. if err != nil {
  787. return ret(err)
  788. }
  789. localMD5 := fmt.Sprintf("\"%x\"", md5.Sum(bs))
  790. if localMD5 != part.ETag {
  791. return ret(errors.New(fmt.Sprintf("CheckSum Failed in Part[%d]", part.PartNumber)))
  792. }
  793. chunks[partNumber].Done = true
  794. chunks[partNumber].ETag = part.ETag
  795. }
  796. return nil
  797. }
  798. // MultiUpload/Upload 为高级upload接口,并发分块上传
  799. //
  800. // 当 partSize > 0 时,由调用者指定分块大小,否则由 SDK 自动切分,单位为MB
  801. // 由调用者指定分块大小时,请确认分块数量不超过10000
  802. //
  803. func (s *ObjectService) MultiUpload(ctx context.Context, name string, filepath string, opt *MultiUploadOptions) (*CompleteMultipartUploadResult, *Response, error) {
  804. return s.Upload(ctx, name, filepath, opt)
  805. }
  806. func (s *ObjectService) Upload(ctx context.Context, name string, filepath string, opt *MultiUploadOptions) (*CompleteMultipartUploadResult, *Response, error) {
  807. if opt == nil {
  808. opt = &MultiUploadOptions{}
  809. }
  810. var localcrc uint64
  811. // 1.Get the file chunk
  812. totalBytes, chunks, partNum, err := SplitFileIntoChunks(filepath, opt.PartSize*1024*1024)
  813. if err != nil {
  814. return nil, nil, err
  815. }
  816. // 校验
  817. if s.client.Conf.EnableCRC {
  818. fd, err := os.Open(filepath)
  819. if err != nil {
  820. return nil, nil, err
  821. }
  822. defer fd.Close()
  823. localcrc, err = calCRC64(fd)
  824. if err != nil {
  825. return nil, nil, err
  826. }
  827. }
  828. // filesize=0 , use simple upload
  829. if partNum == 0 || partNum == 1 {
  830. var opt0 *ObjectPutOptions
  831. if opt.OptIni != nil {
  832. opt0 = &ObjectPutOptions{
  833. opt.OptIni.ACLHeaderOptions,
  834. opt.OptIni.ObjectPutHeaderOptions,
  835. }
  836. }
  837. rsp, err := s.PutFromFile(ctx, name, filepath, opt0)
  838. if err != nil {
  839. return nil, rsp, err
  840. }
  841. result := &CompleteMultipartUploadResult{
  842. Location: fmt.Sprintf("%s/%s", s.client.BaseURL.BucketURL, name),
  843. Key: name,
  844. ETag: rsp.Header.Get("ETag"),
  845. }
  846. if rsp != nil && s.client.Conf.EnableCRC {
  847. scoscrc := rsp.Header.Get("x-cos-hash-crc64ecma")
  848. icoscrc, _ := strconv.ParseUint(scoscrc, 10, 64)
  849. if icoscrc != localcrc {
  850. return result, rsp, fmt.Errorf("verification failed, want:%v, return:%v", localcrc, icoscrc)
  851. }
  852. }
  853. return result, rsp, nil
  854. }
  855. var uploadID string
  856. resumableFlag := false
  857. if opt.CheckPoint {
  858. var err error
  859. uploadID, err = s.getResumableUploadID(ctx, name)
  860. if err == nil && uploadID != "" {
  861. err = s.checkUploadedParts(ctx, name, uploadID, filepath, chunks, partNum)
  862. resumableFlag = (err == nil)
  863. }
  864. }
  865. // 2.Init
  866. optini := opt.OptIni
  867. if !resumableFlag {
  868. res, _, err := s.InitiateMultipartUpload(ctx, name, optini)
  869. if err != nil {
  870. return nil, nil, err
  871. }
  872. uploadID = res.UploadID
  873. }
  874. var poolSize int
  875. if opt.ThreadPoolSize > 0 {
  876. poolSize = opt.ThreadPoolSize
  877. } else {
  878. // Default is one
  879. poolSize = 1
  880. }
  881. chjobs := make(chan *Jobs, 100)
  882. chresults := make(chan *Results, 10000)
  883. optcom := &CompleteMultipartUploadOptions{}
  884. // 3.Start worker
  885. for w := 1; w <= poolSize; w++ {
  886. go worker(s, chjobs, chresults)
  887. }
  888. // progress started event
  889. var listener ProgressListener
  890. var consumedBytes int64
  891. if opt.OptIni != nil {
  892. if opt.OptIni.ObjectPutHeaderOptions != nil {
  893. listener = opt.OptIni.Listener
  894. }
  895. optcom.XOptionHeader, _ = deliverInitOptions(opt.OptIni)
  896. }
  897. event := newProgressEvent(ProgressStartedEvent, 0, 0, totalBytes)
  898. progressCallback(listener, event)
  899. // 4.Push jobs
  900. go func() {
  901. for _, chunk := range chunks {
  902. if chunk.Done {
  903. continue
  904. }
  905. partOpt := &ObjectUploadPartOptions{}
  906. if optini != nil && optini.ObjectPutHeaderOptions != nil {
  907. partOpt.XCosSSECustomerAglo = optini.XCosSSECustomerAglo
  908. partOpt.XCosSSECustomerKey = optini.XCosSSECustomerKey
  909. partOpt.XCosSSECustomerKeyMD5 = optini.XCosSSECustomerKeyMD5
  910. partOpt.XCosTrafficLimit = optini.XCosTrafficLimit
  911. }
  912. job := &Jobs{
  913. Name: name,
  914. RetryTimes: 3,
  915. FilePath: filepath,
  916. UploadId: uploadID,
  917. Chunk: chunk,
  918. Opt: partOpt,
  919. }
  920. chjobs <- job
  921. }
  922. close(chjobs)
  923. }()
  924. // 5.Recv the resp etag to complete
  925. err = nil
  926. for i := 0; i < partNum; i++ {
  927. if chunks[i].Done {
  928. optcom.Parts = append(optcom.Parts, Object{
  929. PartNumber: chunks[i].Number, ETag: chunks[i].ETag},
  930. )
  931. if err == nil {
  932. consumedBytes += chunks[i].Size
  933. event = newProgressEvent(ProgressDataEvent, chunks[i].Size, consumedBytes, totalBytes)
  934. progressCallback(listener, event)
  935. }
  936. continue
  937. }
  938. res := <-chresults
  939. // Notice one part fail can not get the etag according.
  940. if res.Resp == nil || res.err != nil {
  941. // Some part already fail, can not to get the header inside.
  942. err = fmt.Errorf("UploadID %s, part %d failed to get resp content. error: %s", uploadID, res.PartNumber, res.err.Error())
  943. continue
  944. }
  945. // Notice one part fail can not get the etag according.
  946. etag := res.Resp.Header.Get("ETag")
  947. optcom.Parts = append(optcom.Parts, Object{
  948. PartNumber: res.PartNumber, ETag: etag},
  949. )
  950. if err == nil {
  951. consumedBytes += chunks[res.PartNumber-1].Size
  952. event = newProgressEvent(ProgressDataEvent, chunks[res.PartNumber-1].Size, consumedBytes, totalBytes)
  953. progressCallback(listener, event)
  954. }
  955. }
  956. close(chresults)
  957. if err != nil {
  958. event = newProgressEvent(ProgressFailedEvent, 0, consumedBytes, totalBytes, err)
  959. progressCallback(listener, event)
  960. return nil, nil, err
  961. }
  962. sort.Sort(ObjectList(optcom.Parts))
  963. event = newProgressEvent(ProgressCompletedEvent, 0, consumedBytes, totalBytes)
  964. progressCallback(listener, event)
  965. v, resp, err := s.CompleteMultipartUpload(context.Background(), name, uploadID, optcom)
  966. if err != nil {
  967. return v, resp, err
  968. }
  969. if resp != nil && s.client.Conf.EnableCRC {
  970. scoscrc := resp.Header.Get("x-cos-hash-crc64ecma")
  971. icoscrc, _ := strconv.ParseUint(scoscrc, 10, 64)
  972. if icoscrc != localcrc {
  973. return v, resp, fmt.Errorf("verification failed, want:%v, return:%v", localcrc, icoscrc)
  974. }
  975. }
  976. return v, resp, err
  977. }
  978. func SplitSizeIntoChunks(totalBytes int64, partSize int64) ([]Chunk, int, error) {
  979. var partNum int64
  980. if partSize > 0 {
  981. partNum = totalBytes / partSize
  982. if partNum >= 10000 {
  983. return nil, 0, errors.New("Too manry parts, out of 10000")
  984. }
  985. } else {
  986. partNum, partSize = DividePart(totalBytes, 64)
  987. }
  988. var chunks []Chunk
  989. var chunk = Chunk{}
  990. for i := int64(0); i < partNum; i++ {
  991. chunk.Number = int(i + 1)
  992. chunk.OffSet = i * partSize
  993. chunk.Size = partSize
  994. chunks = append(chunks, chunk)
  995. }
  996. if totalBytes%partSize > 0 {
  997. chunk.Number = len(chunks) + 1
  998. chunk.OffSet = int64(len(chunks)) * partSize
  999. chunk.Size = totalBytes % partSize
  1000. chunks = append(chunks, chunk)
  1001. partNum++
  1002. }
  1003. return chunks, int(partNum), nil
  1004. }
  1005. func (s *ObjectService) checkDownloadedParts(opt *MultiDownloadCPInfo, chfile string, chunks []Chunk) (*MultiDownloadCPInfo, bool) {
  1006. var defaultRes MultiDownloadCPInfo
  1007. defaultRes = *opt
  1008. fd, err := os.Open(chfile)
  1009. // checkpoint 文件不存在
  1010. if err != nil && os.IsNotExist(err) {
  1011. // 创建 checkpoint 文件
  1012. fd, _ = os.OpenFile(chfile, os.O_RDONLY|os.O_CREATE|os.O_TRUNC, 0660)
  1013. fd.Close()
  1014. return &defaultRes, false
  1015. }
  1016. if err != nil {
  1017. return &defaultRes, false
  1018. }
  1019. defer fd.Close()
  1020. var res MultiDownloadCPInfo
  1021. err = json.NewDecoder(fd).Decode(&res)
  1022. if err != nil {
  1023. return &defaultRes, false
  1024. }
  1025. // 与COS的文件比较
  1026. if res.CRC64 != opt.CRC64 || res.ETag != opt.ETag || res.Size != opt.Size || res.LastModified != opt.LastModified || len(res.DownloadedBlocks) == 0 {
  1027. return &defaultRes, false
  1028. }
  1029. // len(chunks) 大于1,否则为简单下载, chunks[0].Size为partSize
  1030. partSize := chunks[0].Size
  1031. for _, v := range res.DownloadedBlocks {
  1032. index := v.From / partSize
  1033. to := chunks[index].OffSet + chunks[index].Size - 1
  1034. if chunks[index].OffSet != v.From || to != v.To {
  1035. // 重置chunks
  1036. for i, _ := range chunks {
  1037. chunks[i].Done = false
  1038. }
  1039. return &defaultRes, false
  1040. }
  1041. chunks[index].Done = true
  1042. }
  1043. return &res, true
  1044. }
  1045. func (s *ObjectService) Download(ctx context.Context, name string, filepath string, opt *MultiDownloadOptions, id ...string) (*Response, error) {
  1046. // 参数校验
  1047. if opt == nil {
  1048. opt = &MultiDownloadOptions{}
  1049. }
  1050. if opt.Opt != nil && opt.Opt.Range != "" {
  1051. return nil, fmt.Errorf("Download doesn't support Range Options")
  1052. }
  1053. // 获取文件长度和CRC
  1054. var coscrc string
  1055. resp, err := s.Head(ctx, name, nil, id...)
  1056. if err != nil {
  1057. return resp, err
  1058. }
  1059. // 如果对象不存在x-cos-hash-crc64ecma,则跳过不做校验
  1060. coscrc = resp.Header.Get("x-cos-hash-crc64ecma")
  1061. strTotalBytes := resp.Header.Get("Content-Length")
  1062. totalBytes, err := strconv.ParseInt(strTotalBytes, 10, 64)
  1063. if err != nil {
  1064. return resp, err
  1065. }
  1066. // 切分
  1067. chunks, partNum, err := SplitSizeIntoChunks(totalBytes, opt.PartSize*1024*1024)
  1068. if err != nil {
  1069. return resp, err
  1070. }
  1071. // 直接下载到文件
  1072. if partNum == 0 || partNum == 1 {
  1073. rsp, err := s.GetToFile(ctx, name, filepath, opt.Opt, id...)
  1074. if err != nil {
  1075. return rsp, err
  1076. }
  1077. if coscrc != "" && s.client.Conf.EnableCRC {
  1078. icoscrc, _ := strconv.ParseUint(coscrc, 10, 64)
  1079. fd, err := os.Open(filepath)
  1080. if err != nil {
  1081. return rsp, err
  1082. }
  1083. defer fd.Close()
  1084. localcrc, err := calCRC64(fd)
  1085. if err != nil {
  1086. return rsp, err
  1087. }
  1088. if localcrc != icoscrc {
  1089. return rsp, fmt.Errorf("verification failed, want:%v, return:%v", icoscrc, localcrc)
  1090. }
  1091. }
  1092. return rsp, err
  1093. }
  1094. // 断点续载
  1095. var resumableFlag bool
  1096. var resumableInfo *MultiDownloadCPInfo
  1097. var cpfd *os.File
  1098. var cpfile string
  1099. if opt.CheckPoint {
  1100. cpInfo := &MultiDownloadCPInfo{
  1101. LastModified: resp.Header.Get("Last-Modified"),
  1102. ETag: resp.Header.Get("ETag"),
  1103. CRC64: coscrc,
  1104. Size: totalBytes,
  1105. }
  1106. cpfile = opt.CheckPointFile
  1107. if cpfile == "" {
  1108. cpfile = fmt.Sprintf("%s.cosresumabletask", filepath)
  1109. }
  1110. resumableInfo, resumableFlag = s.checkDownloadedParts(cpInfo, cpfile, chunks)
  1111. cpfd, err = os.OpenFile(cpfile, os.O_RDWR, 0660)
  1112. if err != nil {
  1113. return nil, fmt.Errorf("Open CheckPoint File[%v] Failed:%v", cpfile, err)
  1114. }
  1115. }
  1116. if !resumableFlag {
  1117. // 创建文件
  1118. nfile, err := os.OpenFile(filepath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660)
  1119. if err != nil {
  1120. if cpfd != nil {
  1121. cpfd.Close()
  1122. }
  1123. return resp, err
  1124. }
  1125. nfile.Close()
  1126. }
  1127. var poolSize int
  1128. if opt.ThreadPoolSize > 0 {
  1129. poolSize = opt.ThreadPoolSize
  1130. } else {
  1131. poolSize = 1
  1132. }
  1133. chjobs := make(chan *Jobs, 100)
  1134. chresults := make(chan *Results, 10000)
  1135. for w := 1; w <= poolSize; w++ {
  1136. go downloadWorker(s, chjobs, chresults)
  1137. }
  1138. go func() {
  1139. for _, chunk := range chunks {
  1140. if chunk.Done {
  1141. continue
  1142. }
  1143. var downOpt ObjectGetOptions
  1144. if opt.Opt != nil {
  1145. downOpt = *opt.Opt
  1146. downOpt.Listener = nil // listener need to set nil
  1147. }
  1148. job := &Jobs{
  1149. Name: name,
  1150. RetryTimes: 3,
  1151. FilePath: filepath,
  1152. Chunk: chunk,
  1153. DownOpt: &downOpt,
  1154. }
  1155. if len(id) > 0 {
  1156. job.VersionId = append(job.VersionId, id...)
  1157. }
  1158. chjobs <- job
  1159. }
  1160. close(chjobs)
  1161. }()
  1162. err = nil
  1163. for i := 0; i < partNum; i++ {
  1164. if chunks[i].Done {
  1165. continue
  1166. }
  1167. res := <-chresults
  1168. if res.Resp == nil || res.err != nil {
  1169. err = fmt.Errorf("part %d get resp Content. error: %s", res.PartNumber, res.err.Error())
  1170. continue
  1171. }
  1172. // Dump CheckPoint Info
  1173. if opt.CheckPoint {
  1174. cpfd.Truncate(0)
  1175. cpfd.Seek(0, os.SEEK_SET)
  1176. resumableInfo.DownloadedBlocks = append(resumableInfo.DownloadedBlocks, DownloadedBlock{
  1177. From: chunks[res.PartNumber-1].OffSet,
  1178. To: chunks[res.PartNumber-1].OffSet + chunks[res.PartNumber-1].Size - 1,
  1179. })
  1180. json.NewEncoder(cpfd).Encode(resumableInfo)
  1181. }
  1182. }
  1183. close(chresults)
  1184. if cpfd != nil {
  1185. cpfd.Close()
  1186. }
  1187. if err != nil {
  1188. return nil, err
  1189. }
  1190. // 下载成功,删除checkpoint文件
  1191. if opt.CheckPoint {
  1192. os.Remove(cpfile)
  1193. }
  1194. if coscrc != "" && s.client.Conf.EnableCRC {
  1195. icoscrc, _ := strconv.ParseUint(coscrc, 10, 64)
  1196. fd, err := os.Open(filepath)
  1197. if err != nil {
  1198. return resp, err
  1199. }
  1200. defer fd.Close()
  1201. localcrc, err := calCRC64(fd)
  1202. if err != nil {
  1203. return resp, err
  1204. }
  1205. if localcrc != icoscrc {
  1206. return resp, fmt.Errorf("verification failed, want:%v, return:%v", icoscrc, localcrc)
  1207. }
  1208. }
  1209. return resp, err
  1210. }
  1211. type ObjectPutTaggingOptions struct {
  1212. XMLName xml.Name `xml:"Tagging"`
  1213. TagSet []ObjectTaggingTag `xml:"TagSet>Tag,omitempty"`
  1214. }
  1215. type ObjectTaggingTag BucketTaggingTag
  1216. type ObjectGetTaggingResult ObjectPutTaggingOptions
  1217. func (s *ObjectService) PutTagging(ctx context.Context, name string, opt *ObjectPutTaggingOptions, id ...string) (*Response, error) {
  1218. var u string
  1219. if len(id) == 1 {
  1220. u = fmt.Sprintf("/%s?tagging&versionId=%s", encodeURIComponent(name), id[0])
  1221. } else if len(id) == 0 {
  1222. u = fmt.Sprintf("/%s?tagging", encodeURIComponent(name))
  1223. } else {
  1224. return nil, errors.New("wrong params")
  1225. }
  1226. sendOpt := &sendOptions{
  1227. baseURL: s.client.BaseURL.BucketURL,
  1228. uri: u,
  1229. method: http.MethodPut,
  1230. body: opt,
  1231. }
  1232. resp, err := s.client.send(ctx, sendOpt)
  1233. return resp, err
  1234. }
  1235. func (s *ObjectService) GetTagging(ctx context.Context, name string, id ...string) (*ObjectGetTaggingResult, *Response, error) {
  1236. var u string
  1237. if len(id) == 1 {
  1238. u = fmt.Sprintf("/%s?tagging&versionId=%s", encodeURIComponent(name), id[0])
  1239. } else if len(id) == 0 {
  1240. u = fmt.Sprintf("/%s?tagging", encodeURIComponent(name))
  1241. } else {
  1242. return nil, nil, errors.New("wrong params")
  1243. }
  1244. var res ObjectGetTaggingResult
  1245. sendOpt := &sendOptions{
  1246. baseURL: s.client.BaseURL.BucketURL,
  1247. uri: u,
  1248. method: http.MethodGet,
  1249. result: &res,
  1250. }
  1251. resp, err := s.client.send(ctx, sendOpt)
  1252. return &res, resp, err
  1253. }
  1254. func (s *ObjectService) DeleteTagging(ctx context.Context, name string, id ...string) (*Response, error) {
  1255. var u string
  1256. if len(id) == 1 {
  1257. u = fmt.Sprintf("/%s?tagging&versionId=%s", encodeURIComponent(name), id[0])
  1258. } else if len(id) == 0 {
  1259. u = fmt.Sprintf("/%s?tagging", encodeURIComponent(name))
  1260. } else {
  1261. return nil, errors.New("wrong params")
  1262. }
  1263. sendOpt := &sendOptions{
  1264. baseURL: s.client.BaseURL.BucketURL,
  1265. uri: u,
  1266. method: http.MethodDelete,
  1267. }
  1268. resp, err := s.client.send(ctx, sendOpt)
  1269. return resp, err
  1270. }