You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

1342 lines
41 KiB

4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
  1. package cos
  2. import (
  3. "context"
  4. "crypto/md5"
  5. "encoding/json"
  6. "encoding/xml"
  7. "errors"
  8. "fmt"
  9. "hash/crc64"
  10. "io"
  11. "io/ioutil"
  12. "net/http"
  13. "net/url"
  14. "os"
  15. "sort"
  16. "strconv"
  17. "strings"
  18. "time"
  19. )
  20. // ObjectService 相关 API
  21. type ObjectService service
  22. // ObjectGetOptions is the option of GetObject
  23. type ObjectGetOptions struct {
  24. ResponseContentType string `url:"response-content-type,omitempty" header:"-"`
  25. ResponseContentLanguage string `url:"response-content-language,omitempty" header:"-"`
  26. ResponseExpires string `url:"response-expires,omitempty" header:"-"`
  27. ResponseCacheControl string `url:"response-cache-control,omitempty" header:"-"`
  28. ResponseContentDisposition string `url:"response-content-disposition,omitempty" header:"-"`
  29. ResponseContentEncoding string `url:"response-content-encoding,omitempty" header:"-"`
  30. Range string `url:"-" header:"Range,omitempty"`
  31. IfModifiedSince string `url:"-" header:"If-Modified-Since,omitempty"`
  32. // SSE-C
  33. XCosSSECustomerAglo string `header:"x-cos-server-side-encryption-customer-algorithm,omitempty" url:"-" xml:"-"`
  34. XCosSSECustomerKey string `header:"x-cos-server-side-encryption-customer-key,omitempty" url:"-" xml:"-"`
  35. XCosSSECustomerKeyMD5 string `header:"x-cos-server-side-encryption-customer-key-MD5,omitempty" url:"-" xml:"-"`
  36. //兼容其他自定义头部
  37. XOptionHeader *http.Header `header:"-,omitempty" url:"-" xml:"-"`
  38. XCosTrafficLimit int `header:"x-cos-traffic-limit,omitempty" url:"-" xml:"-"`
  39. // 下载进度, ProgressCompleteEvent不能表示对应API调用成功,API是否调用成功的判断标准为返回err==nil
  40. Listener ProgressListener `header:"-" url:"-" xml:"-"`
  41. }
  42. // presignedURLTestingOptions is the opt of presigned url
  43. type presignedURLTestingOptions struct {
  44. authTime *AuthTime
  45. }
  46. // Get Object 请求可以将一个文件(Object)下载至本地。
  47. // 该操作需要对目标 Object 具有读权限或目标 Object 对所有人都开放了读权限(公有读)。
  48. //
  49. // https://www.qcloud.com/document/product/436/7753
  50. func (s *ObjectService) Get(ctx context.Context, name string, opt *ObjectGetOptions, id ...string) (*Response, error) {
  51. var u string
  52. if len(id) == 1 {
  53. u = fmt.Sprintf("/%s?versionId=%s", encodeURIComponent(name), id[0])
  54. } else if len(id) == 0 {
  55. u = "/" + encodeURIComponent(name)
  56. } else {
  57. return nil, errors.New("wrong params")
  58. }
  59. sendOpt := sendOptions{
  60. baseURL: s.client.BaseURL.BucketURL,
  61. uri: u,
  62. method: http.MethodGet,
  63. optQuery: opt,
  64. optHeader: opt,
  65. disableCloseBody: true,
  66. }
  67. resp, err := s.client.send(ctx, &sendOpt)
  68. if opt != nil && opt.Listener != nil {
  69. if err == nil && resp != nil {
  70. if totalBytes, e := strconv.ParseInt(resp.Header.Get("Content-Length"), 10, 64); e == nil {
  71. resp.Body = TeeReader(resp.Body, nil, totalBytes, opt.Listener)
  72. }
  73. }
  74. }
  75. return resp, err
  76. }
  77. // GetToFile download the object to local file
  78. func (s *ObjectService) GetToFile(ctx context.Context, name, localpath string, opt *ObjectGetOptions, id ...string) (*Response, error) {
  79. resp, err := s.Get(ctx, name, opt, id...)
  80. if err != nil {
  81. return resp, err
  82. }
  83. defer resp.Body.Close()
  84. // If file exist, overwrite it
  85. fd, err := os.OpenFile(localpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660)
  86. if err != nil {
  87. return resp, err
  88. }
  89. _, err = io.Copy(fd, resp.Body)
  90. fd.Close()
  91. if err != nil {
  92. return resp, err
  93. }
  94. return resp, nil
  95. }
  96. type PresignedURLOptions struct {
  97. Query *url.Values `xml:"-" url:"-" header:"-"`
  98. Header *http.Header `header:"-,omitempty" url:"-" xml:"-"`
  99. }
  100. // GetPresignedURL get the object presigned to down or upload file by url
  101. func (s *ObjectService) GetPresignedURL(ctx context.Context, httpMethod, name, ak, sk string, expired time.Duration, opt interface{}) (*url.URL, error) {
  102. sendOpt := sendOptions{
  103. baseURL: s.client.BaseURL.BucketURL,
  104. uri: "/" + encodeURIComponent(name),
  105. method: httpMethod,
  106. optQuery: opt,
  107. optHeader: opt,
  108. }
  109. if popt, ok := opt.(*PresignedURLOptions); ok {
  110. qs := popt.Query.Encode()
  111. if qs != "" {
  112. sendOpt.uri = fmt.Sprintf("%s?%s", sendOpt.uri, qs)
  113. }
  114. }
  115. req, err := s.client.newRequest(ctx, sendOpt.baseURL, sendOpt.uri, sendOpt.method, sendOpt.body, sendOpt.optQuery, sendOpt.optHeader)
  116. if err != nil {
  117. return nil, err
  118. }
  119. var authTime *AuthTime
  120. if opt != nil {
  121. if opt, ok := opt.(*presignedURLTestingOptions); ok {
  122. authTime = opt.authTime
  123. }
  124. }
  125. if authTime == nil {
  126. authTime = NewAuthTime(expired)
  127. }
  128. authorization := newAuthorization(ak, sk, req, authTime)
  129. sign := encodeURIComponent(authorization, []byte{'&', '='})
  130. if req.URL.RawQuery == "" {
  131. req.URL.RawQuery = fmt.Sprintf("%s", sign)
  132. } else {
  133. req.URL.RawQuery = fmt.Sprintf("%s&%s", req.URL.RawQuery, sign)
  134. }
  135. return req.URL, nil
  136. }
  137. // ObjectPutHeaderOptions the options of header of the put object
  138. type ObjectPutHeaderOptions struct {
  139. CacheControl string `header:"Cache-Control,omitempty" url:"-"`
  140. ContentDisposition string `header:"Content-Disposition,omitempty" url:"-"`
  141. ContentEncoding string `header:"Content-Encoding,omitempty" url:"-"`
  142. ContentType string `header:"Content-Type,omitempty" url:"-"`
  143. ContentMD5 string `header:"Content-MD5,omitempty" url:"-"`
  144. ContentLength int64 `header:"Content-Length,omitempty" url:"-"`
  145. ContentLanguage string `header:"Content-Language,omitempty" url:"-"`
  146. Expect string `header:"Expect,omitempty" url:"-"`
  147. Expires string `header:"Expires,omitempty" url:"-"`
  148. XCosContentSHA1 string `header:"x-cos-content-sha1,omitempty" url:"-"`
  149. // 自定义的 x-cos-meta-* header
  150. XCosMetaXXX *http.Header `header:"x-cos-meta-*,omitempty" url:"-"`
  151. XCosStorageClass string `header:"x-cos-storage-class,omitempty" url:"-"`
  152. // 可选值: Normal, Appendable
  153. //XCosObjectType string `header:"x-cos-object-type,omitempty" url:"-"`
  154. // Enable Server Side Encryption, Only supported: AES256
  155. XCosServerSideEncryption string `header:"x-cos-server-side-encryption,omitempty" url:"-" xml:"-"`
  156. // SSE-C
  157. XCosSSECustomerAglo string `header:"x-cos-server-side-encryption-customer-algorithm,omitempty" url:"-" xml:"-"`
  158. XCosSSECustomerKey string `header:"x-cos-server-side-encryption-customer-key,omitempty" url:"-" xml:"-"`
  159. XCosSSECustomerKeyMD5 string `header:"x-cos-server-side-encryption-customer-key-MD5,omitempty" url:"-" xml:"-"`
  160. //兼容其他自定义头部
  161. XOptionHeader *http.Header `header:"-,omitempty" url:"-" xml:"-"`
  162. XCosTrafficLimit int `header:"x-cos-traffic-limit,omitempty" url:"-" xml:"-"`
  163. // 上传进度, ProgressCompleteEvent不能表示对应API调用成功,API是否调用成功的判断标准为返回err==nil
  164. Listener ProgressListener `header:"-" url:"-" xml:"-"`
  165. }
  166. // ObjectPutOptions the options of put object
  167. type ObjectPutOptions struct {
  168. *ACLHeaderOptions `header:",omitempty" url:"-" xml:"-"`
  169. *ObjectPutHeaderOptions `header:",omitempty" url:"-" xml:"-"`
  170. }
  171. // Put Object请求可以将一个文件(Oject)上传至指定Bucket。
  172. //
  173. // https://www.qcloud.com/document/product/436/7749
  174. func (s *ObjectService) Put(ctx context.Context, name string, r io.Reader, uopt *ObjectPutOptions) (*Response, error) {
  175. if r == nil {
  176. return nil, fmt.Errorf("reader is nil")
  177. }
  178. if err := CheckReaderLen(r); err != nil {
  179. return nil, err
  180. }
  181. opt := cloneObjectPutOptions(uopt)
  182. totalBytes, err := GetReaderLen(r)
  183. if err != nil && opt != nil && opt.Listener != nil {
  184. return nil, err
  185. }
  186. if err == nil {
  187. // 与 go http 保持一致, 非bytes.Buffer/bytes.Reader/strings.Reader由用户指定ContentLength, 或使用 Chunk 上传
  188. if opt != nil && opt.ContentLength == 0 && IsLenReader(r) {
  189. opt.ContentLength = totalBytes
  190. }
  191. }
  192. reader := TeeReader(r, nil, totalBytes, nil)
  193. if s.client.Conf.EnableCRC {
  194. reader.writer = crc64.New(crc64.MakeTable(crc64.ECMA))
  195. }
  196. if opt != nil && opt.Listener != nil {
  197. reader.listener = opt.Listener
  198. }
  199. sendOpt := sendOptions{
  200. baseURL: s.client.BaseURL.BucketURL,
  201. uri: "/" + encodeURIComponent(name),
  202. method: http.MethodPut,
  203. body: reader,
  204. optHeader: opt,
  205. }
  206. resp, err := s.client.send(ctx, &sendOpt)
  207. return resp, err
  208. }
  209. // PutFromFile put object from local file
  210. // Notice that when use this put large file need set non-body of debug req/resp, otherwise will out of memory
  211. func (s *ObjectService) PutFromFile(ctx context.Context, name string, filePath string, opt *ObjectPutOptions) (resp *Response, err error) {
  212. nr := 0
  213. for nr < 3 {
  214. fd, e := os.Open(filePath)
  215. if e != nil {
  216. err = e
  217. return
  218. }
  219. resp, err = s.Put(ctx, name, fd, opt)
  220. if err != nil {
  221. nr++
  222. fd.Close()
  223. continue
  224. }
  225. fd.Close()
  226. break
  227. }
  228. return
  229. }
  230. // ObjectCopyHeaderOptions is the head option of the Copy
  231. type ObjectCopyHeaderOptions struct {
  232. // When use replace directive to update meta infos
  233. CacheControl string `header:"Cache-Control,omitempty" url:"-"`
  234. ContentDisposition string `header:"Content-Disposition,omitempty" url:"-"`
  235. ContentEncoding string `header:"Content-Encoding,omitempty" url:"-"`
  236. ContentLanguage string `header:"Content-Language,omitempty" url:"-"`
  237. ContentType string `header:"Content-Type,omitempty" url:"-"`
  238. Expires string `header:"Expires,omitempty" url:"-"`
  239. Expect string `header:"Expect,omitempty" url:"-"`
  240. XCosMetadataDirective string `header:"x-cos-metadata-directive,omitempty" url:"-" xml:"-"`
  241. XCosCopySourceIfModifiedSince string `header:"x-cos-copy-source-If-Modified-Since,omitempty" url:"-" xml:"-"`
  242. XCosCopySourceIfUnmodifiedSince string `header:"x-cos-copy-source-If-Unmodified-Since,omitempty" url:"-" xml:"-"`
  243. XCosCopySourceIfMatch string `header:"x-cos-copy-source-If-Match,omitempty" url:"-" xml:"-"`
  244. XCosCopySourceIfNoneMatch string `header:"x-cos-copy-source-If-None-Match,omitempty" url:"-" xml:"-"`
  245. XCosStorageClass string `header:"x-cos-storage-class,omitempty" url:"-" xml:"-"`
  246. // 自定义的 x-cos-meta-* header
  247. XCosMetaXXX *http.Header `header:"x-cos-meta-*,omitempty" url:"-"`
  248. XCosCopySource string `header:"x-cos-copy-source" url:"-" xml:"-"`
  249. XCosServerSideEncryption string `header:"x-cos-server-side-encryption,omitempty" url:"-" xml:"-"`
  250. // SSE-C
  251. XCosSSECustomerAglo string `header:"x-cos-server-side-encryption-customer-algorithm,omitempty" url:"-" xml:"-"`
  252. XCosSSECustomerKey string `header:"x-cos-server-side-encryption-customer-key,omitempty" url:"-" xml:"-"`
  253. XCosSSECustomerKeyMD5 string `header:"x-cos-server-side-encryption-customer-key-MD5,omitempty" url:"-" xml:"-"`
  254. XCosCopySourceSSECustomerAglo string `header:"x-cos-copy-source-server-side-encryption-customer-algorithm,omitempty" url:"-" xml:"-"`
  255. XCosCopySourceSSECustomerKey string `header:"x-cos-copy-source-server-side-encryption-customer-key,omitempty" url:"-" xml:"-"`
  256. XCosCopySourceSSECustomerKeyMD5 string `header:"x-cos-copy-source-server-side-encryption-customer-key-MD5,omitempty" url:"-" xml:"-"`
  257. //兼容其他自定义头部
  258. XOptionHeader *http.Header `header:"-,omitempty" url:"-" xml:"-"`
  259. }
  260. // ObjectCopyOptions is the option of Copy, choose header or body
  261. type ObjectCopyOptions struct {
  262. *ObjectCopyHeaderOptions `header:",omitempty" url:"-" xml:"-"`
  263. *ACLHeaderOptions `header:",omitempty" url:"-" xml:"-"`
  264. }
  265. // ObjectCopyResult is the result of Copy
  266. type ObjectCopyResult struct {
  267. XMLName xml.Name `xml:"CopyObjectResult"`
  268. ETag string `xml:"ETag,omitempty"`
  269. LastModified string `xml:"LastModified,omitempty"`
  270. CRC64 string `xml:"CRC64,omitempty"`
  271. VersionId string `xml:"VersionId,omitempty"`
  272. }
  273. // Copy 调用 PutObjectCopy 请求实现将一个文件从源路径复制到目标路径。建议文件大小 1M 到 5G,
  274. // 超过 5G 的文件请使用分块上传 Upload - Copy。在拷贝的过程中,文件元属性和 ACL 可以被修改。
  275. //
  276. // 用户可以通过该接口实现文件移动,文件重命名,修改文件属性和创建副本。
  277. //
  278. // 注意:在跨帐号复制的时候,需要先设置被复制文件的权限为公有读,或者对目标帐号赋权,同帐号则不需要。
  279. //
  280. // https://cloud.tencent.com/document/product/436/10881
  281. func (s *ObjectService) Copy(ctx context.Context, name, sourceURL string, opt *ObjectCopyOptions, id ...string) (*ObjectCopyResult, *Response, error) {
  282. surl := strings.SplitN(sourceURL, "/", 2)
  283. if len(surl) < 2 {
  284. return nil, nil, errors.New(fmt.Sprintf("x-cos-copy-source format error: %s", sourceURL))
  285. }
  286. var u string
  287. if len(id) == 1 {
  288. u = fmt.Sprintf("%s/%s?versionId=%s", surl[0], encodeURIComponent(surl[1]), id[0])
  289. } else if len(id) == 0 {
  290. u = fmt.Sprintf("%s/%s", surl[0], encodeURIComponent(surl[1]))
  291. } else {
  292. return nil, nil, errors.New("wrong params")
  293. }
  294. var res ObjectCopyResult
  295. copyOpt := &ObjectCopyOptions{
  296. &ObjectCopyHeaderOptions{},
  297. &ACLHeaderOptions{},
  298. }
  299. if opt != nil {
  300. if opt.ObjectCopyHeaderOptions != nil {
  301. *copyOpt.ObjectCopyHeaderOptions = *opt.ObjectCopyHeaderOptions
  302. }
  303. if opt.ACLHeaderOptions != nil {
  304. *copyOpt.ACLHeaderOptions = *opt.ACLHeaderOptions
  305. }
  306. }
  307. copyOpt.XCosCopySource = u
  308. sendOpt := sendOptions{
  309. baseURL: s.client.BaseURL.BucketURL,
  310. uri: "/" + encodeURIComponent(name),
  311. method: http.MethodPut,
  312. body: nil,
  313. optHeader: copyOpt,
  314. result: &res,
  315. }
  316. resp, err := s.client.send(ctx, &sendOpt)
  317. // If the error occurs during the copy operation, the error response is embedded in the 200 OK response. This means that a 200 OK response can contain either a success or an error.
  318. if err == nil && resp.StatusCode == 200 {
  319. if res.ETag == "" {
  320. return &res, resp, errors.New("response 200 OK, but body contains an error")
  321. }
  322. }
  323. return &res, resp, err
  324. }
  325. type ObjectDeleteOptions struct {
  326. // SSE-C
  327. XCosSSECustomerAglo string `header:"x-cos-server-side-encryption-customer-algorithm,omitempty" url:"-" xml:"-"`
  328. XCosSSECustomerKey string `header:"x-cos-server-side-encryption-customer-key,omitempty" url:"-" xml:"-"`
  329. XCosSSECustomerKeyMD5 string `header:"x-cos-server-side-encryption-customer-key-MD5,omitempty" url:"-" xml:"-"`
  330. //兼容其他自定义头部
  331. XOptionHeader *http.Header `header:"-,omitempty" url:"-" xml:"-"`
  332. VersionId string `header:"-" url:"VersionId,omitempty" xml:"-"`
  333. }
  334. // Delete Object请求可以将一个文件(Object)删除。
  335. //
  336. // https://www.qcloud.com/document/product/436/7743
  337. func (s *ObjectService) Delete(ctx context.Context, name string, opt ...*ObjectDeleteOptions) (*Response, error) {
  338. var optHeader *ObjectDeleteOptions
  339. // When use "" string might call the delete bucket interface
  340. if len(name) == 0 {
  341. return nil, errors.New("empty object name")
  342. }
  343. if len(opt) > 0 {
  344. optHeader = opt[0]
  345. }
  346. sendOpt := sendOptions{
  347. baseURL: s.client.BaseURL.BucketURL,
  348. uri: "/" + encodeURIComponent(name),
  349. method: http.MethodDelete,
  350. optHeader: optHeader,
  351. optQuery: optHeader,
  352. }
  353. resp, err := s.client.send(ctx, &sendOpt)
  354. return resp, err
  355. }
  356. // ObjectHeadOptions is the option of HeadObject
  357. type ObjectHeadOptions struct {
  358. IfModifiedSince string `url:"-" header:"If-Modified-Since,omitempty"`
  359. // SSE-C
  360. XCosSSECustomerAglo string `header:"x-cos-server-side-encryption-customer-algorithm,omitempty" url:"-" xml:"-"`
  361. XCosSSECustomerKey string `header:"x-cos-server-side-encryption-customer-key,omitempty" url:"-" xml:"-"`
  362. XCosSSECustomerKeyMD5 string `header:"x-cos-server-side-encryption-customer-key-MD5,omitempty" url:"-" xml:"-"`
  363. XOptionHeader *http.Header `header:"-,omitempty" url:"-" xml:"-"`
  364. }
  365. // Head Object请求可以取回对应Object的元数据,Head的权限与Get的权限一致
  366. //
  367. // https://www.qcloud.com/document/product/436/7745
  368. func (s *ObjectService) Head(ctx context.Context, name string, opt *ObjectHeadOptions, id ...string) (*Response, error) {
  369. var u string
  370. if len(id) == 1 {
  371. u = fmt.Sprintf("/%s?versionId=%s", encodeURIComponent(name), id[0])
  372. } else if len(id) == 0 {
  373. u = "/" + encodeURIComponent(name)
  374. } else {
  375. return nil, errors.New("wrong params")
  376. }
  377. sendOpt := sendOptions{
  378. baseURL: s.client.BaseURL.BucketURL,
  379. uri: u,
  380. method: http.MethodHead,
  381. optHeader: opt,
  382. }
  383. resp, err := s.client.send(ctx, &sendOpt)
  384. if resp != nil && resp.Header["X-Cos-Object-Type"] != nil && resp.Header["X-Cos-Object-Type"][0] == "appendable" {
  385. resp.Header.Add("x-cos-next-append-position", resp.Header["Content-Length"][0])
  386. }
  387. return resp, err
  388. }
  389. // ObjectOptionsOptions is the option of object options
  390. type ObjectOptionsOptions struct {
  391. Origin string `url:"-" header:"Origin"`
  392. AccessControlRequestMethod string `url:"-" header:"Access-Control-Request-Method"`
  393. AccessControlRequestHeaders string `url:"-" header:"Access-Control-Request-Headers,omitempty"`
  394. }
  395. // Options Object请求实现跨域访问的预请求。即发出一个 OPTIONS 请求给服务器以确认是否可以进行跨域操作。
  396. //
  397. // 当CORS配置不存在时,请求返回403 Forbidden。
  398. //
  399. // https://www.qcloud.com/document/product/436/8288
  400. func (s *ObjectService) Options(ctx context.Context, name string, opt *ObjectOptionsOptions) (*Response, error) {
  401. sendOpt := sendOptions{
  402. baseURL: s.client.BaseURL.BucketURL,
  403. uri: "/" + encodeURIComponent(name),
  404. method: http.MethodOptions,
  405. optHeader: opt,
  406. }
  407. resp, err := s.client.send(ctx, &sendOpt)
  408. return resp, err
  409. }
  410. // CASJobParameters support three way: Standard(in 35 hours), Expedited(quick way, in 15 mins), Bulk(in 5-12 hours_
  411. type CASJobParameters struct {
  412. Tier string `xml:"Tier"`
  413. }
  414. // ObjectRestoreOptions is the option of object restore
  415. type ObjectRestoreOptions struct {
  416. XMLName xml.Name `xml:"RestoreRequest"`
  417. Days int `xml:"Days"`
  418. Tier *CASJobParameters `xml:"CASJobParameters"`
  419. }
  420. // PutRestore API can recover an object of type archived by COS archive.
  421. //
  422. // https://cloud.tencent.com/document/product/436/12633
  423. func (s *ObjectService) PostRestore(ctx context.Context, name string, opt *ObjectRestoreOptions) (*Response, error) {
  424. u := fmt.Sprintf("/%s?restore", encodeURIComponent(name))
  425. sendOpt := sendOptions{
  426. baseURL: s.client.BaseURL.BucketURL,
  427. uri: u,
  428. method: http.MethodPost,
  429. body: opt,
  430. }
  431. resp, err := s.client.send(ctx, &sendOpt)
  432. return resp, err
  433. }
  434. // TODO Append 接口在优化未开放使用
  435. //
  436. // Append请求可以将一个文件(Object)以分块追加的方式上传至 Bucket 中。使用Append Upload的文件必须事前被设定为Appendable。
  437. // 当Appendable的文件被执行Put Object的操作以后,文件被覆盖,属性改变为Normal。
  438. //
  439. // 文件属性可以在Head Object操作中被查询到,当您发起Head Object请求时,会返回自定义Header『x-cos-object-type』,该Header只有两个枚举值:Normal或者Appendable。
  440. //
  441. // 追加上传建议文件大小1M - 5G。如果position的值和当前Object的长度不致,COS会返回409错误。
  442. // 如果Append一个Normal的Object,COS会返回409 ObjectNotAppendable。
  443. //
  444. // Appendable的文件不可以被复制,不参与版本管理,不参与生命周期管理,不可跨区域复制。
  445. //
  446. // 当 r 不是 bytes.Buffer/bytes.Reader/strings.Reader 时,必须指定 opt.ObjectPutHeaderOptions.ContentLength
  447. //
  448. // https://www.qcloud.com/document/product/436/7741
  449. // func (s *ObjectService) Append(ctx context.Context, name string, position int, r io.Reader, opt *ObjectPutOptions) (*Response, error) {
  450. // u := fmt.Sprintf("/%s?append&position=%d", encodeURIComponent(name), position)
  451. // if position != 0{
  452. // opt = nil
  453. // }
  454. // sendOpt := sendOptions{
  455. // baseURL: s.client.BaseURL.BucketURL,
  456. // uri: u,
  457. // method: http.MethodPost,
  458. // optHeader: opt,
  459. // body: r,
  460. // }
  461. // resp, err := s.client.send(ctx, &sendOpt)
  462. // return resp, err
  463. // }
  464. // ObjectDeleteMultiOptions is the option of DeleteMulti
  465. type ObjectDeleteMultiOptions struct {
  466. XMLName xml.Name `xml:"Delete" header:"-"`
  467. Quiet bool `xml:"Quiet" header:"-"`
  468. Objects []Object `xml:"Object" header:"-"`
  469. //XCosSha1 string `xml:"-" header:"x-cos-sha1"`
  470. }
  471. // ObjectDeleteMultiResult is the result of DeleteMulti
  472. type ObjectDeleteMultiResult struct {
  473. XMLName xml.Name `xml:"DeleteResult"`
  474. DeletedObjects []Object `xml:"Deleted,omitempty"`
  475. Errors []struct {
  476. Key string `xml:",omitempty"`
  477. Code string `xml:",omitempty"`
  478. Message string `xml:",omitempty"`
  479. VersionId string `xml:",omitempty"`
  480. } `xml:"Error,omitempty"`
  481. }
  482. // DeleteMulti 请求实现批量删除文件,最大支持单次删除1000个文件。
  483. // 对于返回结果,COS提供Verbose和Quiet两种结果模式。Verbose模式将返回每个Object的删除结果;
  484. // Quiet模式只返回报错的Object信息。
  485. // https://www.qcloud.com/document/product/436/8289
  486. func (s *ObjectService) DeleteMulti(ctx context.Context, opt *ObjectDeleteMultiOptions) (*ObjectDeleteMultiResult, *Response, error) {
  487. var res ObjectDeleteMultiResult
  488. sendOpt := sendOptions{
  489. baseURL: s.client.BaseURL.BucketURL,
  490. uri: "/?delete",
  491. method: http.MethodPost,
  492. body: opt,
  493. result: &res,
  494. }
  495. resp, err := s.client.send(ctx, &sendOpt)
  496. return &res, resp, err
  497. }
  498. // Object is the meta info of the object
  499. type Object struct {
  500. Key string `xml:",omitempty"`
  501. ETag string `xml:",omitempty"`
  502. Size int64 `xml:",omitempty"`
  503. PartNumber int `xml:",omitempty"`
  504. LastModified string `xml:",omitempty"`
  505. StorageClass string `xml:",omitempty"`
  506. Owner *Owner `xml:",omitempty"`
  507. VersionId string `xml:",omitempty"`
  508. }
  509. // MultiUploadOptions is the option of the multiupload,
  510. // ThreadPoolSize default is one
  511. type MultiUploadOptions struct {
  512. OptIni *InitiateMultipartUploadOptions
  513. PartSize int64
  514. ThreadPoolSize int
  515. CheckPoint bool
  516. EnableVerification bool
  517. }
  518. type MultiDownloadOptions struct {
  519. Opt *ObjectGetOptions
  520. PartSize int64
  521. ThreadPoolSize int
  522. CheckPoint bool
  523. CheckPointFile string
  524. }
  525. type MultiDownloadCPInfo struct {
  526. Size int64 `json:"contentLength,omitempty"`
  527. ETag string `json:"eTag,omitempty"`
  528. CRC64 string `json:"crc64ecma,omitempty"`
  529. LastModified string `json:"lastModified,omitempty"`
  530. DownloadedBlocks []DownloadedBlock `json:"downloadedBlocks,omitempty"`
  531. }
  532. type DownloadedBlock struct {
  533. From int64 `json:"from,omitempty"`
  534. To int64 `json:"to,omitempty"`
  535. }
  536. type Chunk struct {
  537. Number int
  538. OffSet int64
  539. Size int64
  540. Done bool
  541. ETag string
  542. }
  543. // jobs
  544. type Jobs struct {
  545. Name string
  546. UploadId string
  547. FilePath string
  548. RetryTimes int
  549. VersionId []string
  550. Chunk Chunk
  551. Data io.Reader
  552. Opt *ObjectUploadPartOptions
  553. DownOpt *ObjectGetOptions
  554. }
  555. type Results struct {
  556. PartNumber int
  557. Resp *Response
  558. err error
  559. }
  560. func LimitReadCloser(r io.Reader, n int64) io.Reader {
  561. var lc LimitedReadCloser
  562. lc.R = r
  563. lc.N = n
  564. return &lc
  565. }
  566. type LimitedReadCloser struct {
  567. io.LimitedReader
  568. }
  569. func (lc *LimitedReadCloser) Close() error {
  570. if r, ok := lc.R.(io.ReadCloser); ok {
  571. return r.Close()
  572. }
  573. return nil
  574. }
  575. func worker(s *ObjectService, jobs <-chan *Jobs, results chan<- *Results) {
  576. for j := range jobs {
  577. j.Opt.ContentLength = j.Chunk.Size
  578. rt := j.RetryTimes
  579. for {
  580. // http.Request.Body can be Closed in request
  581. fd, err := os.Open(j.FilePath)
  582. var res Results
  583. if err != nil {
  584. res.err = err
  585. res.PartNumber = j.Chunk.Number
  586. res.Resp = nil
  587. results <- &res
  588. break
  589. }
  590. fd.Seek(j.Chunk.OffSet, os.SEEK_SET)
  591. resp, err := s.UploadPart(context.Background(), j.Name, j.UploadId, j.Chunk.Number,
  592. LimitReadCloser(fd, j.Chunk.Size), j.Opt)
  593. res.PartNumber = j.Chunk.Number
  594. res.Resp = resp
  595. res.err = err
  596. if err != nil {
  597. rt--
  598. if rt == 0 {
  599. results <- &res
  600. break
  601. }
  602. continue
  603. }
  604. results <- &res
  605. break
  606. }
  607. }
  608. }
  609. func downloadWorker(s *ObjectService, jobs <-chan *Jobs, results chan<- *Results) {
  610. for j := range jobs {
  611. opt := &RangeOptions{
  612. HasStart: true,
  613. HasEnd: true,
  614. Start: j.Chunk.OffSet,
  615. End: j.Chunk.OffSet + j.Chunk.Size - 1,
  616. }
  617. j.DownOpt.Range = FormatRangeOptions(opt)
  618. rt := j.RetryTimes
  619. for {
  620. var res Results
  621. res.PartNumber = j.Chunk.Number
  622. resp, err := s.Get(context.Background(), j.Name, j.DownOpt, j.VersionId...)
  623. res.err = err
  624. res.Resp = resp
  625. if err != nil {
  626. rt--
  627. if rt == 0 {
  628. results <- &res
  629. break
  630. }
  631. continue
  632. }
  633. defer resp.Body.Close()
  634. fd, err := os.OpenFile(j.FilePath, os.O_WRONLY, 0660)
  635. if err != nil {
  636. res.err = err
  637. results <- &res
  638. break
  639. }
  640. fd.Seek(j.Chunk.OffSet, os.SEEK_SET)
  641. n, err := io.Copy(fd, LimitReadCloser(resp.Body, j.Chunk.Size))
  642. if n != j.Chunk.Size || err != nil {
  643. res.err = fmt.Errorf("io.Copy Failed, nread:%v, want:%v, err:%v", n, j.Chunk.Size, err)
  644. }
  645. fd.Close()
  646. results <- &res
  647. break
  648. }
  649. }
  650. }
  651. func DividePart(fileSize int64, last int) (int64, int64) {
  652. partSize := int64(last * 1024 * 1024)
  653. partNum := fileSize / partSize
  654. for partNum >= 10000 {
  655. partSize = partSize * 2
  656. partNum = fileSize / partSize
  657. }
  658. return partNum, partSize
  659. }
  660. func SplitFileIntoChunks(filePath string, partSize int64) (int64, []Chunk, int, error) {
  661. if filePath == "" {
  662. return 0, nil, 0, errors.New("filePath invalid")
  663. }
  664. file, err := os.Open(filePath)
  665. if err != nil {
  666. return 0, nil, 0, err
  667. }
  668. defer file.Close()
  669. stat, err := file.Stat()
  670. if err != nil {
  671. return 0, nil, 0, err
  672. }
  673. var partNum int64
  674. if partSize > 0 {
  675. partSize = partSize * 1024 * 1024
  676. partNum = stat.Size() / partSize
  677. if partNum >= 10000 {
  678. return 0, nil, 0, errors.New("Too many parts, out of 10000")
  679. }
  680. } else {
  681. partNum, partSize = DividePart(stat.Size(), 64)
  682. }
  683. var chunks []Chunk
  684. var chunk = Chunk{}
  685. for i := int64(0); i < partNum; i++ {
  686. chunk.Number = int(i + 1)
  687. chunk.OffSet = i * partSize
  688. chunk.Size = partSize
  689. chunks = append(chunks, chunk)
  690. }
  691. if stat.Size()%partSize > 0 {
  692. chunk.Number = len(chunks) + 1
  693. chunk.OffSet = int64(len(chunks)) * partSize
  694. chunk.Size = stat.Size() % partSize
  695. chunks = append(chunks, chunk)
  696. partNum++
  697. }
  698. return int64(stat.Size()), chunks, int(partNum), nil
  699. }
  700. func (s *ObjectService) getResumableUploadID(ctx context.Context, name string) (string, error) {
  701. opt := &ObjectListUploadsOptions{
  702. Prefix: name,
  703. EncodingType: "url",
  704. }
  705. res, _, err := s.ListUploads(ctx, opt)
  706. if err != nil {
  707. return "", err
  708. }
  709. if len(res.Upload) == 0 {
  710. return "", nil
  711. }
  712. last := len(res.Upload) - 1
  713. for last >= 0 {
  714. decodeKey, _ := decodeURIComponent(res.Upload[last].Key)
  715. if decodeKey == name {
  716. return decodeURIComponent(res.Upload[last].UploadID)
  717. }
  718. last = last - 1
  719. }
  720. return "", nil
  721. }
  722. func (s *ObjectService) checkUploadedParts(ctx context.Context, name, UploadID, filepath string, chunks []Chunk, partNum int) error {
  723. var uploadedParts []Object
  724. isTruncated := true
  725. opt := &ObjectListPartsOptions{
  726. EncodingType: "url",
  727. }
  728. for isTruncated {
  729. res, _, err := s.ListParts(ctx, name, UploadID, opt)
  730. if err != nil {
  731. return err
  732. }
  733. if len(res.Parts) > 0 {
  734. uploadedParts = append(uploadedParts, res.Parts...)
  735. }
  736. isTruncated = res.IsTruncated
  737. opt.PartNumberMarker = res.NextPartNumberMarker
  738. }
  739. fd, err := os.Open(filepath)
  740. if err != nil {
  741. return err
  742. }
  743. defer fd.Close()
  744. // 某个分块出错, 重置chunks
  745. ret := func(e error) error {
  746. for i, _ := range chunks {
  747. chunks[i].Done = false
  748. chunks[i].ETag = ""
  749. }
  750. return e
  751. }
  752. for _, part := range uploadedParts {
  753. partNumber := part.PartNumber
  754. if partNumber > partNum {
  755. return ret(errors.New("Part Number is not consistent"))
  756. }
  757. partNumber = partNumber - 1
  758. fd.Seek(chunks[partNumber].OffSet, os.SEEK_SET)
  759. bs, err := ioutil.ReadAll(io.LimitReader(fd, chunks[partNumber].Size))
  760. if err != nil {
  761. return ret(err)
  762. }
  763. localMD5 := fmt.Sprintf("\"%x\"", md5.Sum(bs))
  764. if localMD5 != part.ETag {
  765. return ret(errors.New(fmt.Sprintf("CheckSum Failed in Part[%d]", part.PartNumber)))
  766. }
  767. chunks[partNumber].Done = true
  768. chunks[partNumber].ETag = part.ETag
  769. }
  770. return nil
  771. }
  772. // MultiUpload/Upload 为高级upload接口,并发分块上传
  773. //
  774. // 当 partSize > 0 时,由调用者指定分块大小,否则由 SDK 自动切分,单位为MB
  775. // 由调用者指定分块大小时,请确认分块数量不超过10000
  776. //
  777. func (s *ObjectService) MultiUpload(ctx context.Context, name string, filepath string, opt *MultiUploadOptions) (*CompleteMultipartUploadResult, *Response, error) {
  778. return s.Upload(ctx, name, filepath, opt)
  779. }
  780. func (s *ObjectService) Upload(ctx context.Context, name string, filepath string, opt *MultiUploadOptions) (*CompleteMultipartUploadResult, *Response, error) {
  781. if opt == nil {
  782. opt = &MultiUploadOptions{}
  783. }
  784. var localcrc uint64
  785. // 1.Get the file chunk
  786. totalBytes, chunks, partNum, err := SplitFileIntoChunks(filepath, opt.PartSize)
  787. if err != nil {
  788. return nil, nil, err
  789. }
  790. // 校验
  791. if s.client.Conf.EnableCRC {
  792. fd, err := os.Open(filepath)
  793. if err != nil {
  794. return nil, nil, err
  795. }
  796. defer fd.Close()
  797. localcrc, err = calCRC64(fd)
  798. if err != nil {
  799. return nil, nil, err
  800. }
  801. }
  802. // filesize=0 , use simple upload
  803. if partNum == 0 || partNum == 1 {
  804. var opt0 *ObjectPutOptions
  805. if opt.OptIni != nil {
  806. opt0 = &ObjectPutOptions{
  807. opt.OptIni.ACLHeaderOptions,
  808. opt.OptIni.ObjectPutHeaderOptions,
  809. }
  810. }
  811. rsp, err := s.PutFromFile(ctx, name, filepath, opt0)
  812. if err != nil {
  813. return nil, rsp, err
  814. }
  815. result := &CompleteMultipartUploadResult{
  816. Location: fmt.Sprintf("%s/%s", s.client.BaseURL.BucketURL, name),
  817. Key: name,
  818. ETag: rsp.Header.Get("ETag"),
  819. }
  820. if rsp != nil && s.client.Conf.EnableCRC {
  821. scoscrc := rsp.Header.Get("x-cos-hash-crc64ecma")
  822. icoscrc, _ := strconv.ParseUint(scoscrc, 10, 64)
  823. if icoscrc != localcrc {
  824. return result, rsp, fmt.Errorf("verification failed, want:%v, return:%v", localcrc, icoscrc)
  825. }
  826. }
  827. return result, rsp, nil
  828. }
  829. var uploadID string
  830. resumableFlag := false
  831. if opt.CheckPoint {
  832. var err error
  833. uploadID, err = s.getResumableUploadID(ctx, name)
  834. if err == nil && uploadID != "" {
  835. err = s.checkUploadedParts(ctx, name, uploadID, filepath, chunks, partNum)
  836. resumableFlag = (err == nil)
  837. }
  838. }
  839. // 2.Init
  840. optini := opt.OptIni
  841. if !resumableFlag {
  842. res, _, err := s.InitiateMultipartUpload(ctx, name, optini)
  843. if err != nil {
  844. return nil, nil, err
  845. }
  846. uploadID = res.UploadID
  847. }
  848. var poolSize int
  849. if opt.ThreadPoolSize > 0 {
  850. poolSize = opt.ThreadPoolSize
  851. } else {
  852. // Default is one
  853. poolSize = 1
  854. }
  855. chjobs := make(chan *Jobs, 100)
  856. chresults := make(chan *Results, 10000)
  857. optcom := &CompleteMultipartUploadOptions{}
  858. // 3.Start worker
  859. for w := 1; w <= poolSize; w++ {
  860. go worker(s, chjobs, chresults)
  861. }
  862. // progress started event
  863. var listener ProgressListener
  864. var consumedBytes int64
  865. if opt.OptIni != nil {
  866. listener = opt.OptIni.Listener
  867. optcom.XOptionHeader, _ = deliverInitOptions(opt.OptIni)
  868. }
  869. event := newProgressEvent(ProgressStartedEvent, 0, 0, totalBytes)
  870. progressCallback(listener, event)
  871. // 4.Push jobs
  872. go func() {
  873. for _, chunk := range chunks {
  874. if chunk.Done {
  875. continue
  876. }
  877. partOpt := &ObjectUploadPartOptions{}
  878. if optini != nil && optini.ObjectPutHeaderOptions != nil {
  879. partOpt.XCosSSECustomerAglo = optini.XCosSSECustomerAglo
  880. partOpt.XCosSSECustomerKey = optini.XCosSSECustomerKey
  881. partOpt.XCosSSECustomerKeyMD5 = optini.XCosSSECustomerKeyMD5
  882. partOpt.XCosTrafficLimit = optini.XCosTrafficLimit
  883. }
  884. job := &Jobs{
  885. Name: name,
  886. RetryTimes: 3,
  887. FilePath: filepath,
  888. UploadId: uploadID,
  889. Chunk: chunk,
  890. Opt: partOpt,
  891. }
  892. chjobs <- job
  893. }
  894. close(chjobs)
  895. }()
  896. // 5.Recv the resp etag to complete
  897. err = nil
  898. for i := 0; i < partNum; i++ {
  899. if chunks[i].Done {
  900. optcom.Parts = append(optcom.Parts, Object{
  901. PartNumber: chunks[i].Number, ETag: chunks[i].ETag},
  902. )
  903. if err == nil {
  904. consumedBytes += chunks[i].Size
  905. event = newProgressEvent(ProgressDataEvent, chunks[i].Size, consumedBytes, totalBytes)
  906. progressCallback(listener, event)
  907. }
  908. continue
  909. }
  910. res := <-chresults
  911. // Notice one part fail can not get the etag according.
  912. if res.Resp == nil || res.err != nil {
  913. // Some part already fail, can not to get the header inside.
  914. err = fmt.Errorf("UploadID %s, part %d failed to get resp content. error: %s", uploadID, res.PartNumber, res.err.Error())
  915. continue
  916. }
  917. // Notice one part fail can not get the etag according.
  918. etag := res.Resp.Header.Get("ETag")
  919. optcom.Parts = append(optcom.Parts, Object{
  920. PartNumber: res.PartNumber, ETag: etag},
  921. )
  922. if err == nil {
  923. consumedBytes += chunks[res.PartNumber-1].Size
  924. event = newProgressEvent(ProgressDataEvent, chunks[res.PartNumber-1].Size, consumedBytes, totalBytes)
  925. progressCallback(listener, event)
  926. }
  927. }
  928. close(chresults)
  929. if err != nil {
  930. event = newProgressEvent(ProgressFailedEvent, 0, consumedBytes, totalBytes, err)
  931. progressCallback(listener, event)
  932. return nil, nil, err
  933. }
  934. sort.Sort(ObjectList(optcom.Parts))
  935. event = newProgressEvent(ProgressCompletedEvent, 0, consumedBytes, totalBytes)
  936. progressCallback(listener, event)
  937. v, resp, err := s.CompleteMultipartUpload(context.Background(), name, uploadID, optcom)
  938. if err != nil {
  939. return v, resp, err
  940. }
  941. if resp != nil && s.client.Conf.EnableCRC {
  942. scoscrc := resp.Header.Get("x-cos-hash-crc64ecma")
  943. icoscrc, _ := strconv.ParseUint(scoscrc, 10, 64)
  944. if icoscrc != localcrc {
  945. return v, resp, fmt.Errorf("verification failed, want:%v, return:%v", localcrc, icoscrc)
  946. }
  947. }
  948. return v, resp, err
  949. }
  950. func SplitSizeIntoChunks(totalBytes int64, partSize int64) ([]Chunk, int, error) {
  951. var partNum int64
  952. if partSize > 0 {
  953. partSize = partSize * 1024 * 1024
  954. partNum = totalBytes / partSize
  955. if partNum >= 10000 {
  956. return nil, 0, errors.New("Too manry parts, out of 10000")
  957. }
  958. } else {
  959. partNum, partSize = DividePart(totalBytes, 64)
  960. }
  961. var chunks []Chunk
  962. var chunk = Chunk{}
  963. for i := int64(0); i < partNum; i++ {
  964. chunk.Number = int(i + 1)
  965. chunk.OffSet = i * partSize
  966. chunk.Size = partSize
  967. chunks = append(chunks, chunk)
  968. }
  969. if totalBytes%partSize > 0 {
  970. chunk.Number = len(chunks) + 1
  971. chunk.OffSet = int64(len(chunks)) * partSize
  972. chunk.Size = totalBytes % partSize
  973. chunks = append(chunks, chunk)
  974. partNum++
  975. }
  976. return chunks, int(partNum), nil
  977. }
  978. func (s *ObjectService) checkDownloadedParts(opt *MultiDownloadCPInfo, chfile string, chunks []Chunk) (*MultiDownloadCPInfo, bool) {
  979. var defaultRes MultiDownloadCPInfo
  980. defaultRes = *opt
  981. fd, err := os.Open(chfile)
  982. // checkpoint 文件不存在
  983. if err != nil && os.IsNotExist(err) {
  984. // 创建 checkpoint 文件
  985. fd, _ = os.OpenFile(chfile, os.O_RDONLY|os.O_CREATE|os.O_TRUNC, 0660)
  986. fd.Close()
  987. return &defaultRes, false
  988. }
  989. if err != nil {
  990. return &defaultRes, false
  991. }
  992. defer fd.Close()
  993. var res MultiDownloadCPInfo
  994. err = json.NewDecoder(fd).Decode(&res)
  995. if err != nil {
  996. return &defaultRes, false
  997. }
  998. // 与COS的文件比较
  999. if res.CRC64 != opt.CRC64 || res.ETag != opt.ETag || res.Size != opt.Size || res.LastModified != opt.LastModified || len(res.DownloadedBlocks) == 0 {
  1000. return &defaultRes, false
  1001. }
  1002. // len(chunks) 大于1,否则为简单下载, chunks[0].Size为partSize
  1003. partSize := chunks[0].Size
  1004. for _, v := range res.DownloadedBlocks {
  1005. index := v.From / partSize
  1006. to := chunks[index].OffSet + chunks[index].Size - 1
  1007. if chunks[index].OffSet != v.From || to != v.To {
  1008. // 重置chunks
  1009. for i, _ := range chunks {
  1010. chunks[i].Done = false
  1011. }
  1012. return &defaultRes, false
  1013. }
  1014. chunks[index].Done = true
  1015. }
  1016. return &res, true
  1017. }
  1018. func (s *ObjectService) Download(ctx context.Context, name string, filepath string, opt *MultiDownloadOptions, id ...string) (*Response, error) {
  1019. // 参数校验
  1020. if opt == nil {
  1021. opt = &MultiDownloadOptions{}
  1022. }
  1023. if opt.Opt != nil && opt.Opt.Range != "" {
  1024. return nil, fmt.Errorf("Download doesn't support Range Options")
  1025. }
  1026. // 获取文件长度和CRC
  1027. var coscrc string
  1028. resp, err := s.Head(ctx, name, nil, id...)
  1029. if err != nil {
  1030. return resp, err
  1031. }
  1032. // 如果对象不存在x-cos-hash-crc64ecma,则跳过不做校验
  1033. coscrc = resp.Header.Get("x-cos-hash-crc64ecma")
  1034. strTotalBytes := resp.Header.Get("Content-Length")
  1035. totalBytes, err := strconv.ParseInt(strTotalBytes, 10, 64)
  1036. if err != nil {
  1037. return resp, err
  1038. }
  1039. // 切分
  1040. chunks, partNum, err := SplitSizeIntoChunks(totalBytes, opt.PartSize)
  1041. if err != nil {
  1042. return resp, err
  1043. }
  1044. // 直接下载到文件
  1045. if partNum == 0 || partNum == 1 {
  1046. rsp, err := s.GetToFile(ctx, name, filepath, opt.Opt, id...)
  1047. if err != nil {
  1048. return rsp, err
  1049. }
  1050. if coscrc != "" && s.client.Conf.EnableCRC {
  1051. icoscrc, _ := strconv.ParseUint(coscrc, 10, 64)
  1052. fd, err := os.Open(filepath)
  1053. if err != nil {
  1054. return rsp, err
  1055. }
  1056. defer fd.Close()
  1057. localcrc, err := calCRC64(fd)
  1058. if err != nil {
  1059. return rsp, err
  1060. }
  1061. if localcrc != icoscrc {
  1062. return rsp, fmt.Errorf("verification failed, want:%v, return:%v", icoscrc, localcrc)
  1063. }
  1064. }
  1065. return rsp, err
  1066. }
  1067. // 断点续载
  1068. var resumableFlag bool
  1069. var resumableInfo *MultiDownloadCPInfo
  1070. var cpfd *os.File
  1071. var cpfile string
  1072. if opt.CheckPoint {
  1073. cpInfo := &MultiDownloadCPInfo{
  1074. LastModified: resp.Header.Get("Last-Modified"),
  1075. ETag: resp.Header.Get("ETag"),
  1076. CRC64: coscrc,
  1077. Size: totalBytes,
  1078. }
  1079. cpfile = opt.CheckPointFile
  1080. if cpfile == "" {
  1081. cpfile = fmt.Sprintf("%s.cosresumabletask", filepath)
  1082. }
  1083. resumableInfo, resumableFlag = s.checkDownloadedParts(cpInfo, cpfile, chunks)
  1084. cpfd, err = os.OpenFile(cpfile, os.O_RDWR, 0660)
  1085. if err != nil {
  1086. return nil, fmt.Errorf("Open CheckPoint File[%v] Failed:%v", cpfile, err)
  1087. }
  1088. }
  1089. if !resumableFlag {
  1090. // 创建文件
  1091. nfile, err := os.OpenFile(filepath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660)
  1092. if err != nil {
  1093. if cpfd != nil {
  1094. cpfd.Close()
  1095. }
  1096. return resp, err
  1097. }
  1098. nfile.Close()
  1099. }
  1100. var poolSize int
  1101. if opt.ThreadPoolSize > 0 {
  1102. poolSize = opt.ThreadPoolSize
  1103. } else {
  1104. poolSize = 1
  1105. }
  1106. chjobs := make(chan *Jobs, 100)
  1107. chresults := make(chan *Results, 10000)
  1108. for w := 1; w <= poolSize; w++ {
  1109. go downloadWorker(s, chjobs, chresults)
  1110. }
  1111. go func() {
  1112. for _, chunk := range chunks {
  1113. if chunk.Done {
  1114. continue
  1115. }
  1116. var downOpt ObjectGetOptions
  1117. if opt.Opt != nil {
  1118. downOpt = *opt.Opt
  1119. downOpt.Listener = nil // listener need to set nil
  1120. }
  1121. job := &Jobs{
  1122. Name: name,
  1123. RetryTimes: 3,
  1124. FilePath: filepath,
  1125. Chunk: chunk,
  1126. DownOpt: &downOpt,
  1127. }
  1128. if len(id) > 0 {
  1129. job.VersionId = append(job.VersionId, id...)
  1130. }
  1131. chjobs <- job
  1132. }
  1133. close(chjobs)
  1134. }()
  1135. err = nil
  1136. for i := 0; i < partNum; i++ {
  1137. if chunks[i].Done {
  1138. continue
  1139. }
  1140. res := <-chresults
  1141. if res.Resp == nil || res.err != nil {
  1142. err = fmt.Errorf("part %d get resp Content. error: %s", res.PartNumber, res.err.Error())
  1143. continue
  1144. }
  1145. // Dump CheckPoint Info
  1146. if opt.CheckPoint {
  1147. cpfd.Truncate(0)
  1148. cpfd.Seek(0, os.SEEK_SET)
  1149. resumableInfo.DownloadedBlocks = append(resumableInfo.DownloadedBlocks, DownloadedBlock{
  1150. From: chunks[res.PartNumber-1].OffSet,
  1151. To: chunks[res.PartNumber-1].OffSet + chunks[res.PartNumber-1].Size - 1,
  1152. })
  1153. json.NewEncoder(cpfd).Encode(resumableInfo)
  1154. }
  1155. }
  1156. close(chresults)
  1157. if cpfd != nil {
  1158. cpfd.Close()
  1159. }
  1160. if err != nil {
  1161. return nil, err
  1162. }
  1163. // 下载成功,删除checkpoint文件
  1164. if opt.CheckPoint {
  1165. os.Remove(cpfile)
  1166. }
  1167. if coscrc != "" && s.client.Conf.EnableCRC {
  1168. icoscrc, _ := strconv.ParseUint(coscrc, 10, 64)
  1169. fd, err := os.Open(filepath)
  1170. if err != nil {
  1171. return resp, err
  1172. }
  1173. defer fd.Close()
  1174. localcrc, err := calCRC64(fd)
  1175. if err != nil {
  1176. return resp, err
  1177. }
  1178. if localcrc != icoscrc {
  1179. return resp, fmt.Errorf("verification failed, want:%v, return:%v", icoscrc, localcrc)
  1180. }
  1181. }
  1182. return resp, err
  1183. }
  1184. type ObjectPutTaggingOptions struct {
  1185. XMLName xml.Name `xml:"Tagging"`
  1186. TagSet []ObjectTaggingTag `xml:"TagSet>Tag,omitempty"`
  1187. }
  1188. type ObjectTaggingTag BucketTaggingTag
  1189. type ObjectGetTaggingResult ObjectPutTaggingOptions
  1190. func (s *ObjectService) PutTagging(ctx context.Context, name string, opt *ObjectPutTaggingOptions, id ...string) (*Response, error) {
  1191. var u string
  1192. if len(id) == 1 {
  1193. u = fmt.Sprintf("/%s?tagging&versionId=%s", encodeURIComponent(name), id[0])
  1194. } else if len(id) == 0 {
  1195. u = fmt.Sprintf("/%s?tagging", encodeURIComponent(name))
  1196. } else {
  1197. return nil, errors.New("wrong params")
  1198. }
  1199. sendOpt := &sendOptions{
  1200. baseURL: s.client.BaseURL.BucketURL,
  1201. uri: u,
  1202. method: http.MethodPut,
  1203. body: opt,
  1204. }
  1205. resp, err := s.client.send(ctx, sendOpt)
  1206. return resp, err
  1207. }
  1208. func (s *ObjectService) GetTagging(ctx context.Context, name string, id ...string) (*ObjectGetTaggingResult, *Response, error) {
  1209. var u string
  1210. if len(id) == 1 {
  1211. u = fmt.Sprintf("/%s?tagging&versionId=%s", encodeURIComponent(name), id[0])
  1212. } else if len(id) == 0 {
  1213. u = fmt.Sprintf("/%s?tagging", encodeURIComponent(name))
  1214. } else {
  1215. return nil, nil, errors.New("wrong params")
  1216. }
  1217. var res ObjectGetTaggingResult
  1218. sendOpt := &sendOptions{
  1219. baseURL: s.client.BaseURL.BucketURL,
  1220. uri: u,
  1221. method: http.MethodGet,
  1222. result: &res,
  1223. }
  1224. resp, err := s.client.send(ctx, sendOpt)
  1225. return &res, resp, err
  1226. }
  1227. func (s *ObjectService) DeleteTagging(ctx context.Context, name string, id ...string) (*Response, error) {
  1228. var u string
  1229. if len(id) == 1 {
  1230. u = fmt.Sprintf("/%s?tagging&versionId=%s", encodeURIComponent(name), id[0])
  1231. } else if len(id) == 0 {
  1232. u = fmt.Sprintf("/%s?tagging", encodeURIComponent(name))
  1233. } else {
  1234. return nil, errors.New("wrong params")
  1235. }
  1236. sendOpt := &sendOptions{
  1237. baseURL: s.client.BaseURL.BucketURL,
  1238. uri: u,
  1239. method: http.MethodDelete,
  1240. }
  1241. resp, err := s.client.send(ctx, sendOpt)
  1242. return resp, err
  1243. }