You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

1384 lines
42 KiB

4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
4 years ago
  1. package cos
  2. import (
  3. "context"
  4. "crypto/md5"
  5. "encoding/json"
  6. "encoding/xml"
  7. "errors"
  8. "fmt"
  9. "hash/crc64"
  10. "io"
  11. "io/ioutil"
  12. "net/http"
  13. "net/url"
  14. "os"
  15. "sort"
  16. "strconv"
  17. "strings"
  18. "time"
  19. )
  20. // ObjectService 相关 API
  21. type ObjectService service
  22. // ObjectGetOptions is the option of GetObject
  23. type ObjectGetOptions struct {
  24. ResponseContentType string `url:"response-content-type,omitempty" header:"-"`
  25. ResponseContentLanguage string `url:"response-content-language,omitempty" header:"-"`
  26. ResponseExpires string `url:"response-expires,omitempty" header:"-"`
  27. ResponseCacheControl string `url:"response-cache-control,omitempty" header:"-"`
  28. ResponseContentDisposition string `url:"response-content-disposition,omitempty" header:"-"`
  29. ResponseContentEncoding string `url:"response-content-encoding,omitempty" header:"-"`
  30. Range string `url:"-" header:"Range,omitempty"`
  31. IfModifiedSince string `url:"-" header:"If-Modified-Since,omitempty"`
  32. // SSE-C
  33. XCosSSECustomerAglo string `header:"x-cos-server-side-encryption-customer-algorithm,omitempty" url:"-" xml:"-"`
  34. XCosSSECustomerKey string `header:"x-cos-server-side-encryption-customer-key,omitempty" url:"-" xml:"-"`
  35. XCosSSECustomerKeyMD5 string `header:"x-cos-server-side-encryption-customer-key-MD5,omitempty" url:"-" xml:"-"`
  36. //兼容其他自定义头部
  37. XOptionHeader *http.Header `header:"-,omitempty" url:"-" xml:"-"`
  38. XCosTrafficLimit int `header:"x-cos-traffic-limit,omitempty" url:"-" xml:"-"`
  39. // 下载进度, ProgressCompleteEvent不能表示对应API调用成功,API是否调用成功的判断标准为返回err==nil
  40. Listener ProgressListener `header:"-" url:"-" xml:"-"`
  41. }
  42. // presignedURLTestingOptions is the opt of presigned url
  43. type presignedURLTestingOptions struct {
  44. authTime *AuthTime
  45. }
  46. // Get Object 请求可以将一个文件(Object)下载至本地。
  47. // 该操作需要对目标 Object 具有读权限或目标 Object 对所有人都开放了读权限(公有读)。
  48. //
  49. // https://www.qcloud.com/document/product/436/7753
  50. func (s *ObjectService) Get(ctx context.Context, name string, opt *ObjectGetOptions, id ...string) (*Response, error) {
  51. var u string
  52. if len(id) == 1 {
  53. u = fmt.Sprintf("/%s?versionId=%s", encodeURIComponent(name), id[0])
  54. } else if len(id) == 0 {
  55. u = "/" + encodeURIComponent(name)
  56. } else {
  57. return nil, errors.New("wrong params")
  58. }
  59. sendOpt := sendOptions{
  60. baseURL: s.client.BaseURL.BucketURL,
  61. uri: u,
  62. method: http.MethodGet,
  63. optQuery: opt,
  64. optHeader: opt,
  65. disableCloseBody: true,
  66. }
  67. resp, err := s.client.send(ctx, &sendOpt)
  68. if opt != nil && opt.Listener != nil {
  69. if err == nil && resp != nil {
  70. if totalBytes, e := strconv.ParseInt(resp.Header.Get("Content-Length"), 10, 64); e == nil {
  71. resp.Body = TeeReader(resp.Body, nil, totalBytes, opt.Listener)
  72. }
  73. }
  74. }
  75. return resp, err
  76. }
  77. // GetToFile download the object to local file
  78. func (s *ObjectService) GetToFile(ctx context.Context, name, localpath string, opt *ObjectGetOptions, id ...string) (*Response, error) {
  79. resp, err := s.Get(ctx, name, opt, id...)
  80. if err != nil {
  81. return resp, err
  82. }
  83. defer resp.Body.Close()
  84. // If file exist, overwrite it
  85. fd, err := os.OpenFile(localpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660)
  86. if err != nil {
  87. return resp, err
  88. }
  89. _, err = io.Copy(fd, resp.Body)
  90. fd.Close()
  91. if err != nil {
  92. return resp, err
  93. }
  94. return resp, nil
  95. }
  96. func (s *ObjectService) GetObjectURL(name string) *url.URL {
  97. uri, _ := url.Parse("/" + encodeURIComponent(name, []byte{'/'}))
  98. return s.client.BaseURL.BucketURL.ResolveReference(uri)
  99. }
  100. type PresignedURLOptions struct {
  101. Query *url.Values `xml:"-" url:"-" header:"-"`
  102. Header *http.Header `header:"-,omitempty" url:"-" xml:"-"`
  103. }
  104. // GetPresignedURL get the object presigned to down or upload file by url
  105. func (s *ObjectService) GetPresignedURL(ctx context.Context, httpMethod, name, ak, sk string, expired time.Duration, opt interface{}) (*url.URL, error) {
  106. sendOpt := sendOptions{
  107. baseURL: s.client.BaseURL.BucketURL,
  108. uri: "/" + encodeURIComponent(name),
  109. method: httpMethod,
  110. optQuery: opt,
  111. optHeader: opt,
  112. }
  113. if popt, ok := opt.(*PresignedURLOptions); ok {
  114. qs := popt.Query.Encode()
  115. if qs != "" {
  116. sendOpt.uri = fmt.Sprintf("%s?%s", sendOpt.uri, qs)
  117. }
  118. }
  119. req, err := s.client.newRequest(ctx, sendOpt.baseURL, sendOpt.uri, sendOpt.method, sendOpt.body, sendOpt.optQuery, sendOpt.optHeader)
  120. if err != nil {
  121. return nil, err
  122. }
  123. var authTime *AuthTime
  124. if opt != nil {
  125. if opt, ok := opt.(*presignedURLTestingOptions); ok {
  126. authTime = opt.authTime
  127. }
  128. }
  129. if authTime == nil {
  130. authTime = NewAuthTime(expired)
  131. }
  132. authorization := newAuthorization(ak, sk, req, authTime)
  133. sign := encodeURIComponent(authorization, []byte{'&', '='})
  134. if req.URL.RawQuery == "" {
  135. req.URL.RawQuery = fmt.Sprintf("%s", sign)
  136. } else {
  137. req.URL.RawQuery = fmt.Sprintf("%s&%s", req.URL.RawQuery, sign)
  138. }
  139. return req.URL, nil
  140. }
  141. // ObjectPutHeaderOptions the options of header of the put object
  142. type ObjectPutHeaderOptions struct {
  143. CacheControl string `header:"Cache-Control,omitempty" url:"-"`
  144. ContentDisposition string `header:"Content-Disposition,omitempty" url:"-"`
  145. ContentEncoding string `header:"Content-Encoding,omitempty" url:"-"`
  146. ContentType string `header:"Content-Type,omitempty" url:"-"`
  147. ContentMD5 string `header:"Content-MD5,omitempty" url:"-"`
  148. ContentLength int64 `header:"Content-Length,omitempty" url:"-"`
  149. ContentLanguage string `header:"Content-Language,omitempty" url:"-"`
  150. Expect string `header:"Expect,omitempty" url:"-"`
  151. Expires string `header:"Expires,omitempty" url:"-"`
  152. XCosContentSHA1 string `header:"x-cos-content-sha1,omitempty" url:"-"`
  153. // 自定义的 x-cos-meta-* header
  154. XCosMetaXXX *http.Header `header:"x-cos-meta-*,omitempty" url:"-"`
  155. XCosStorageClass string `header:"x-cos-storage-class,omitempty" url:"-"`
  156. // 可选值: Normal, Appendable
  157. //XCosObjectType string `header:"x-cos-object-type,omitempty" url:"-"`
  158. // Enable Server Side Encryption, Only supported: AES256
  159. XCosServerSideEncryption string `header:"x-cos-server-side-encryption,omitempty" url:"-" xml:"-"`
  160. // SSE-C
  161. XCosSSECustomerAglo string `header:"x-cos-server-side-encryption-customer-algorithm,omitempty" url:"-" xml:"-"`
  162. XCosSSECustomerKey string `header:"x-cos-server-side-encryption-customer-key,omitempty" url:"-" xml:"-"`
  163. XCosSSECustomerKeyMD5 string `header:"x-cos-server-side-encryption-customer-key-MD5,omitempty" url:"-" xml:"-"`
  164. //兼容其他自定义头部
  165. XOptionHeader *http.Header `header:"-,omitempty" url:"-" xml:"-"`
  166. XCosTrafficLimit int `header:"x-cos-traffic-limit,omitempty" url:"-" xml:"-"`
  167. // 上传进度, ProgressCompleteEvent不能表示对应API调用成功,API是否调用成功的判断标准为返回err==nil
  168. Listener ProgressListener `header:"-" url:"-" xml:"-"`
  169. }
  170. // ObjectPutOptions the options of put object
  171. type ObjectPutOptions struct {
  172. *ACLHeaderOptions `header:",omitempty" url:"-" xml:"-"`
  173. *ObjectPutHeaderOptions `header:",omitempty" url:"-" xml:"-"`
  174. }
  175. // Put Object请求可以将一个文件(Oject)上传至指定Bucket。
  176. //
  177. // https://www.qcloud.com/document/product/436/7749
  178. func (s *ObjectService) Put(ctx context.Context, name string, r io.Reader, uopt *ObjectPutOptions) (*Response, error) {
  179. if r == nil {
  180. return nil, fmt.Errorf("reader is nil")
  181. }
  182. if err := CheckReaderLen(r); err != nil {
  183. return nil, err
  184. }
  185. opt := CloneObjectPutOptions(uopt)
  186. totalBytes, err := GetReaderLen(r)
  187. if err != nil && opt != nil && opt.Listener != nil {
  188. if opt.ContentLength == 0 {
  189. return nil, err
  190. }
  191. totalBytes = opt.ContentLength
  192. }
  193. if err == nil {
  194. // 与 go http 保持一致, 非bytes.Buffer/bytes.Reader/strings.Reader由用户指定ContentLength, 或使用 Chunk 上传
  195. if opt != nil && opt.ContentLength == 0 && IsLenReader(r) {
  196. opt.ContentLength = totalBytes
  197. }
  198. }
  199. reader := TeeReader(r, nil, totalBytes, nil)
  200. if s.client.Conf.EnableCRC {
  201. reader.writer = crc64.New(crc64.MakeTable(crc64.ECMA))
  202. }
  203. if opt != nil && opt.Listener != nil {
  204. reader.listener = opt.Listener
  205. }
  206. sendOpt := sendOptions{
  207. baseURL: s.client.BaseURL.BucketURL,
  208. uri: "/" + encodeURIComponent(name),
  209. method: http.MethodPut,
  210. body: reader,
  211. optHeader: opt,
  212. }
  213. resp, err := s.client.send(ctx, &sendOpt)
  214. return resp, err
  215. }
  216. // PutFromFile put object from local file
  217. // Notice that when use this put large file need set non-body of debug req/resp, otherwise will out of memory
  218. func (s *ObjectService) PutFromFile(ctx context.Context, name string, filePath string, opt *ObjectPutOptions) (resp *Response, err error) {
  219. nr := 0
  220. for nr < 3 {
  221. fd, e := os.Open(filePath)
  222. if e != nil {
  223. err = e
  224. return
  225. }
  226. resp, err = s.Put(ctx, name, fd, opt)
  227. if err != nil {
  228. nr++
  229. fd.Close()
  230. continue
  231. }
  232. fd.Close()
  233. break
  234. }
  235. return
  236. }
  237. // ObjectCopyHeaderOptions is the head option of the Copy
  238. type ObjectCopyHeaderOptions struct {
  239. // When use replace directive to update meta infos
  240. CacheControl string `header:"Cache-Control,omitempty" url:"-"`
  241. ContentDisposition string `header:"Content-Disposition,omitempty" url:"-"`
  242. ContentEncoding string `header:"Content-Encoding,omitempty" url:"-"`
  243. ContentLanguage string `header:"Content-Language,omitempty" url:"-"`
  244. ContentType string `header:"Content-Type,omitempty" url:"-"`
  245. Expires string `header:"Expires,omitempty" url:"-"`
  246. Expect string `header:"Expect,omitempty" url:"-"`
  247. XCosMetadataDirective string `header:"x-cos-metadata-directive,omitempty" url:"-" xml:"-"`
  248. XCosCopySourceIfModifiedSince string `header:"x-cos-copy-source-If-Modified-Since,omitempty" url:"-" xml:"-"`
  249. XCosCopySourceIfUnmodifiedSince string `header:"x-cos-copy-source-If-Unmodified-Since,omitempty" url:"-" xml:"-"`
  250. XCosCopySourceIfMatch string `header:"x-cos-copy-source-If-Match,omitempty" url:"-" xml:"-"`
  251. XCosCopySourceIfNoneMatch string `header:"x-cos-copy-source-If-None-Match,omitempty" url:"-" xml:"-"`
  252. XCosStorageClass string `header:"x-cos-storage-class,omitempty" url:"-" xml:"-"`
  253. // 自定义的 x-cos-meta-* header
  254. XCosMetaXXX *http.Header `header:"x-cos-meta-*,omitempty" url:"-"`
  255. XCosCopySource string `header:"x-cos-copy-source" url:"-" xml:"-"`
  256. XCosServerSideEncryption string `header:"x-cos-server-side-encryption,omitempty" url:"-" xml:"-"`
  257. // SSE-C
  258. XCosSSECustomerAglo string `header:"x-cos-server-side-encryption-customer-algorithm,omitempty" url:"-" xml:"-"`
  259. XCosSSECustomerKey string `header:"x-cos-server-side-encryption-customer-key,omitempty" url:"-" xml:"-"`
  260. XCosSSECustomerKeyMD5 string `header:"x-cos-server-side-encryption-customer-key-MD5,omitempty" url:"-" xml:"-"`
  261. XCosCopySourceSSECustomerAglo string `header:"x-cos-copy-source-server-side-encryption-customer-algorithm,omitempty" url:"-" xml:"-"`
  262. XCosCopySourceSSECustomerKey string `header:"x-cos-copy-source-server-side-encryption-customer-key,omitempty" url:"-" xml:"-"`
  263. XCosCopySourceSSECustomerKeyMD5 string `header:"x-cos-copy-source-server-side-encryption-customer-key-MD5,omitempty" url:"-" xml:"-"`
  264. //兼容其他自定义头部
  265. XOptionHeader *http.Header `header:"-,omitempty" url:"-" xml:"-"`
  266. }
  267. // ObjectCopyOptions is the option of Copy, choose header or body
  268. type ObjectCopyOptions struct {
  269. *ObjectCopyHeaderOptions `header:",omitempty" url:"-" xml:"-"`
  270. *ACLHeaderOptions `header:",omitempty" url:"-" xml:"-"`
  271. }
  272. // ObjectCopyResult is the result of Copy
  273. type ObjectCopyResult struct {
  274. XMLName xml.Name `xml:"CopyObjectResult"`
  275. ETag string `xml:"ETag,omitempty"`
  276. LastModified string `xml:"LastModified,omitempty"`
  277. CRC64 string `xml:"CRC64,omitempty"`
  278. VersionId string `xml:"VersionId,omitempty"`
  279. }
  280. // Copy 调用 PutObjectCopy 请求实现将一个文件从源路径复制到目标路径。建议文件大小 1M 到 5G,
  281. // 超过 5G 的文件请使用分块上传 Upload - Copy。在拷贝的过程中,文件元属性和 ACL 可以被修改。
  282. //
  283. // 用户可以通过该接口实现文件移动,文件重命名,修改文件属性和创建副本。
  284. //
  285. // 注意:在跨帐号复制的时候,需要先设置被复制文件的权限为公有读,或者对目标帐号赋权,同帐号则不需要。
  286. //
  287. // https://cloud.tencent.com/document/product/436/10881
  288. func (s *ObjectService) Copy(ctx context.Context, name, sourceURL string, opt *ObjectCopyOptions, id ...string) (*ObjectCopyResult, *Response, error) {
  289. surl := strings.SplitN(sourceURL, "/", 2)
  290. if len(surl) < 2 {
  291. return nil, nil, errors.New(fmt.Sprintf("x-cos-copy-source format error: %s", sourceURL))
  292. }
  293. var u string
  294. if len(id) == 1 {
  295. u = fmt.Sprintf("%s/%s?versionId=%s", surl[0], encodeURIComponent(surl[1]), id[0])
  296. } else if len(id) == 0 {
  297. u = fmt.Sprintf("%s/%s", surl[0], encodeURIComponent(surl[1]))
  298. } else {
  299. return nil, nil, errors.New("wrong params")
  300. }
  301. var res ObjectCopyResult
  302. copyOpt := &ObjectCopyOptions{
  303. &ObjectCopyHeaderOptions{},
  304. &ACLHeaderOptions{},
  305. }
  306. if opt != nil {
  307. if opt.ObjectCopyHeaderOptions != nil {
  308. *copyOpt.ObjectCopyHeaderOptions = *opt.ObjectCopyHeaderOptions
  309. }
  310. if opt.ACLHeaderOptions != nil {
  311. *copyOpt.ACLHeaderOptions = *opt.ACLHeaderOptions
  312. }
  313. }
  314. copyOpt.XCosCopySource = u
  315. sendOpt := sendOptions{
  316. baseURL: s.client.BaseURL.BucketURL,
  317. uri: "/" + encodeURIComponent(name),
  318. method: http.MethodPut,
  319. body: nil,
  320. optHeader: copyOpt,
  321. result: &res,
  322. }
  323. resp, err := s.client.send(ctx, &sendOpt)
  324. // If the error occurs during the copy operation, the error response is embedded in the 200 OK response. This means that a 200 OK response can contain either a success or an error.
  325. if err == nil && resp.StatusCode == 200 {
  326. if res.ETag == "" {
  327. return &res, resp, errors.New("response 200 OK, but body contains an error")
  328. }
  329. }
  330. return &res, resp, err
  331. }
  332. type ObjectDeleteOptions struct {
  333. // SSE-C
  334. XCosSSECustomerAglo string `header:"x-cos-server-side-encryption-customer-algorithm,omitempty" url:"-" xml:"-"`
  335. XCosSSECustomerKey string `header:"x-cos-server-side-encryption-customer-key,omitempty" url:"-" xml:"-"`
  336. XCosSSECustomerKeyMD5 string `header:"x-cos-server-side-encryption-customer-key-MD5,omitempty" url:"-" xml:"-"`
  337. //兼容其他自定义头部
  338. XOptionHeader *http.Header `header:"-,omitempty" url:"-" xml:"-"`
  339. VersionId string `header:"-" url:"VersionId,omitempty" xml:"-"`
  340. }
  341. // Delete Object请求可以将一个文件(Object)删除。
  342. //
  343. // https://www.qcloud.com/document/product/436/7743
  344. func (s *ObjectService) Delete(ctx context.Context, name string, opt ...*ObjectDeleteOptions) (*Response, error) {
  345. var optHeader *ObjectDeleteOptions
  346. // When use "" string might call the delete bucket interface
  347. if len(name) == 0 {
  348. return nil, errors.New("empty object name")
  349. }
  350. if len(opt) > 0 {
  351. optHeader = opt[0]
  352. }
  353. sendOpt := sendOptions{
  354. baseURL: s.client.BaseURL.BucketURL,
  355. uri: "/" + encodeURIComponent(name),
  356. method: http.MethodDelete,
  357. optHeader: optHeader,
  358. optQuery: optHeader,
  359. }
  360. resp, err := s.client.send(ctx, &sendOpt)
  361. return resp, err
  362. }
  363. // ObjectHeadOptions is the option of HeadObject
  364. type ObjectHeadOptions struct {
  365. IfModifiedSince string `url:"-" header:"If-Modified-Since,omitempty"`
  366. // SSE-C
  367. XCosSSECustomerAglo string `header:"x-cos-server-side-encryption-customer-algorithm,omitempty" url:"-" xml:"-"`
  368. XCosSSECustomerKey string `header:"x-cos-server-side-encryption-customer-key,omitempty" url:"-" xml:"-"`
  369. XCosSSECustomerKeyMD5 string `header:"x-cos-server-side-encryption-customer-key-MD5,omitempty" url:"-" xml:"-"`
  370. XOptionHeader *http.Header `header:"-,omitempty" url:"-" xml:"-"`
  371. }
  372. // Head Object请求可以取回对应Object的元数据,Head的权限与Get的权限一致
  373. //
  374. // https://www.qcloud.com/document/product/436/7745
  375. func (s *ObjectService) Head(ctx context.Context, name string, opt *ObjectHeadOptions, id ...string) (*Response, error) {
  376. var u string
  377. if len(id) == 1 {
  378. u = fmt.Sprintf("/%s?versionId=%s", encodeURIComponent(name), id[0])
  379. } else if len(id) == 0 {
  380. u = "/" + encodeURIComponent(name)
  381. } else {
  382. return nil, errors.New("wrong params")
  383. }
  384. sendOpt := sendOptions{
  385. baseURL: s.client.BaseURL.BucketURL,
  386. uri: u,
  387. method: http.MethodHead,
  388. optHeader: opt,
  389. }
  390. resp, err := s.client.send(ctx, &sendOpt)
  391. if resp != nil && resp.Header["X-Cos-Object-Type"] != nil && resp.Header["X-Cos-Object-Type"][0] == "appendable" {
  392. resp.Header.Add("x-cos-next-append-position", resp.Header["Content-Length"][0])
  393. }
  394. return resp, err
  395. }
  396. // ObjectOptionsOptions is the option of object options
  397. type ObjectOptionsOptions struct {
  398. Origin string `url:"-" header:"Origin"`
  399. AccessControlRequestMethod string `url:"-" header:"Access-Control-Request-Method"`
  400. AccessControlRequestHeaders string `url:"-" header:"Access-Control-Request-Headers,omitempty"`
  401. }
  402. // Options Object请求实现跨域访问的预请求。即发出一个 OPTIONS 请求给服务器以确认是否可以进行跨域操作。
  403. //
  404. // 当CORS配置不存在时,请求返回403 Forbidden。
  405. //
  406. // https://www.qcloud.com/document/product/436/8288
  407. func (s *ObjectService) Options(ctx context.Context, name string, opt *ObjectOptionsOptions) (*Response, error) {
  408. sendOpt := sendOptions{
  409. baseURL: s.client.BaseURL.BucketURL,
  410. uri: "/" + encodeURIComponent(name),
  411. method: http.MethodOptions,
  412. optHeader: opt,
  413. }
  414. resp, err := s.client.send(ctx, &sendOpt)
  415. return resp, err
  416. }
  417. // CASJobParameters support three way: Standard(in 35 hours), Expedited(quick way, in 15 mins), Bulk(in 5-12 hours_
  418. type CASJobParameters struct {
  419. Tier string `xml:"Tier"`
  420. }
  421. // ObjectRestoreOptions is the option of object restore
  422. type ObjectRestoreOptions struct {
  423. XMLName xml.Name `xml:"RestoreRequest"`
  424. Days int `xml:"Days"`
  425. Tier *CASJobParameters `xml:"CASJobParameters"`
  426. }
  427. // PutRestore API can recover an object of type archived by COS archive.
  428. //
  429. // https://cloud.tencent.com/document/product/436/12633
  430. func (s *ObjectService) PostRestore(ctx context.Context, name string, opt *ObjectRestoreOptions) (*Response, error) {
  431. u := fmt.Sprintf("/%s?restore", encodeURIComponent(name))
  432. sendOpt := sendOptions{
  433. baseURL: s.client.BaseURL.BucketURL,
  434. uri: u,
  435. method: http.MethodPost,
  436. body: opt,
  437. }
  438. resp, err := s.client.send(ctx, &sendOpt)
  439. return resp, err
  440. }
  441. // TODO Append 接口在优化未开放使用
  442. //
  443. // Append请求可以将一个文件(Object)以分块追加的方式上传至 Bucket 中。使用Append Upload的文件必须事前被设定为Appendable。
  444. // 当Appendable的文件被执行Put Object的操作以后,文件被覆盖,属性改变为Normal。
  445. //
  446. // 文件属性可以在Head Object操作中被查询到,当您发起Head Object请求时,会返回自定义Header『x-cos-object-type』,该Header只有两个枚举值:Normal或者Appendable。
  447. //
  448. // 追加上传建议文件大小1M - 5G。如果position的值和当前Object的长度不致,COS会返回409错误。
  449. // 如果Append一个Normal的Object,COS会返回409 ObjectNotAppendable。
  450. //
  451. // Appendable的文件不可以被复制,不参与版本管理,不参与生命周期管理,不可跨区域复制。
  452. //
  453. // 当 r 不是 bytes.Buffer/bytes.Reader/strings.Reader 时,必须指定 opt.ObjectPutHeaderOptions.ContentLength
  454. //
  455. // https://www.qcloud.com/document/product/436/7741
  456. // func (s *ObjectService) Append(ctx context.Context, name string, position int, r io.Reader, opt *ObjectPutOptions) (*Response, error) {
  457. // u := fmt.Sprintf("/%s?append&position=%d", encodeURIComponent(name), position)
  458. // if position != 0{
  459. // opt = nil
  460. // }
  461. // sendOpt := sendOptions{
  462. // baseURL: s.client.BaseURL.BucketURL,
  463. // uri: u,
  464. // method: http.MethodPost,
  465. // optHeader: opt,
  466. // body: r,
  467. // }
  468. // resp, err := s.client.send(ctx, &sendOpt)
  469. // return resp, err
  470. // }
  471. // ObjectDeleteMultiOptions is the option of DeleteMulti
  472. type ObjectDeleteMultiOptions struct {
  473. XMLName xml.Name `xml:"Delete" header:"-"`
  474. Quiet bool `xml:"Quiet" header:"-"`
  475. Objects []Object `xml:"Object" header:"-"`
  476. //XCosSha1 string `xml:"-" header:"x-cos-sha1"`
  477. }
  478. // ObjectDeleteMultiResult is the result of DeleteMulti
  479. type ObjectDeleteMultiResult struct {
  480. XMLName xml.Name `xml:"DeleteResult"`
  481. DeletedObjects []Object `xml:"Deleted,omitempty"`
  482. Errors []struct {
  483. Key string `xml:",omitempty"`
  484. Code string `xml:",omitempty"`
  485. Message string `xml:",omitempty"`
  486. VersionId string `xml:",omitempty"`
  487. } `xml:"Error,omitempty"`
  488. }
  489. // DeleteMulti 请求实现批量删除文件,最大支持单次删除1000个文件。
  490. // 对于返回结果,COS提供Verbose和Quiet两种结果模式。Verbose模式将返回每个Object的删除结果;
  491. // Quiet模式只返回报错的Object信息。
  492. // https://www.qcloud.com/document/product/436/8289
  493. func (s *ObjectService) DeleteMulti(ctx context.Context, opt *ObjectDeleteMultiOptions) (*ObjectDeleteMultiResult, *Response, error) {
  494. var res ObjectDeleteMultiResult
  495. sendOpt := sendOptions{
  496. baseURL: s.client.BaseURL.BucketURL,
  497. uri: "/?delete",
  498. method: http.MethodPost,
  499. body: opt,
  500. result: &res,
  501. }
  502. resp, err := s.client.send(ctx, &sendOpt)
  503. return &res, resp, err
  504. }
  505. // Object is the meta info of the object
  506. type Object struct {
  507. Key string `xml:",omitempty"`
  508. ETag string `xml:",omitempty"`
  509. Size int64 `xml:",omitempty"`
  510. PartNumber int `xml:",omitempty"`
  511. LastModified string `xml:",omitempty"`
  512. StorageClass string `xml:",omitempty"`
  513. Owner *Owner `xml:",omitempty"`
  514. VersionId string `xml:",omitempty"`
  515. }
  516. // MultiUploadOptions is the option of the multiupload,
  517. // ThreadPoolSize default is one
  518. type MultiUploadOptions struct {
  519. OptIni *InitiateMultipartUploadOptions
  520. PartSize int64
  521. ThreadPoolSize int
  522. CheckPoint bool
  523. EnableVerification bool
  524. }
  525. type MultiDownloadOptions struct {
  526. Opt *ObjectGetOptions
  527. PartSize int64
  528. ThreadPoolSize int
  529. CheckPoint bool
  530. CheckPointFile string
  531. }
  532. type MultiDownloadCPInfo struct {
  533. Size int64 `json:"contentLength,omitempty"`
  534. ETag string `json:"eTag,omitempty"`
  535. CRC64 string `json:"crc64ecma,omitempty"`
  536. LastModified string `json:"lastModified,omitempty"`
  537. DownloadedBlocks []DownloadedBlock `json:"downloadedBlocks,omitempty"`
  538. }
  539. type DownloadedBlock struct {
  540. From int64 `json:"from,omitempty"`
  541. To int64 `json:"to,omitempty"`
  542. }
  543. type Chunk struct {
  544. Number int
  545. OffSet int64
  546. Size int64
  547. Done bool
  548. ETag string
  549. }
  550. // jobs
  551. type Jobs struct {
  552. Name string
  553. UploadId string
  554. FilePath string
  555. RetryTimes int
  556. VersionId []string
  557. Chunk Chunk
  558. Data io.Reader
  559. Opt *ObjectUploadPartOptions
  560. DownOpt *ObjectGetOptions
  561. }
  562. type Results struct {
  563. PartNumber int
  564. Resp *Response
  565. err error
  566. }
  567. func LimitReadCloser(r io.Reader, n int64) io.Reader {
  568. var lc LimitedReadCloser
  569. lc.R = r
  570. lc.N = n
  571. return &lc
  572. }
  573. type LimitedReadCloser struct {
  574. io.LimitedReader
  575. }
  576. func (lc *LimitedReadCloser) Close() error {
  577. if r, ok := lc.R.(io.ReadCloser); ok {
  578. return r.Close()
  579. }
  580. return nil
  581. }
  582. type DiscardReadCloser struct {
  583. RC io.ReadCloser
  584. Discard int
  585. }
  586. func (drc *DiscardReadCloser) Read(data []byte) (int, error) {
  587. n, err := drc.RC.Read(data)
  588. if drc.Discard == 0 || n <= 0 {
  589. return n, err
  590. }
  591. if n <= drc.Discard {
  592. drc.Discard -= n
  593. return 0, err
  594. }
  595. realLen := n - drc.Discard
  596. copy(data[0:realLen], data[drc.Discard:n])
  597. drc.Discard = 0
  598. return realLen, err
  599. }
  600. func (drc *DiscardReadCloser) Close() error {
  601. if rc, ok := drc.RC.(io.ReadCloser); ok {
  602. return rc.Close()
  603. }
  604. return nil
  605. }
  606. func worker(s *ObjectService, jobs <-chan *Jobs, results chan<- *Results) {
  607. for j := range jobs {
  608. j.Opt.ContentLength = j.Chunk.Size
  609. rt := j.RetryTimes
  610. for {
  611. // http.Request.Body can be Closed in request
  612. fd, err := os.Open(j.FilePath)
  613. var res Results
  614. if err != nil {
  615. res.err = err
  616. res.PartNumber = j.Chunk.Number
  617. res.Resp = nil
  618. results <- &res
  619. break
  620. }
  621. fd.Seek(j.Chunk.OffSet, os.SEEK_SET)
  622. resp, err := s.UploadPart(context.Background(), j.Name, j.UploadId, j.Chunk.Number,
  623. LimitReadCloser(fd, j.Chunk.Size), j.Opt)
  624. res.PartNumber = j.Chunk.Number
  625. res.Resp = resp
  626. res.err = err
  627. if err != nil {
  628. rt--
  629. if rt == 0 {
  630. results <- &res
  631. break
  632. }
  633. continue
  634. }
  635. results <- &res
  636. break
  637. }
  638. }
  639. }
  640. func downloadWorker(s *ObjectService, jobs <-chan *Jobs, results chan<- *Results) {
  641. for j := range jobs {
  642. opt := &RangeOptions{
  643. HasStart: true,
  644. HasEnd: true,
  645. Start: j.Chunk.OffSet,
  646. End: j.Chunk.OffSet + j.Chunk.Size - 1,
  647. }
  648. j.DownOpt.Range = FormatRangeOptions(opt)
  649. rt := j.RetryTimes
  650. for {
  651. var res Results
  652. res.PartNumber = j.Chunk.Number
  653. resp, err := s.Get(context.Background(), j.Name, j.DownOpt, j.VersionId...)
  654. res.err = err
  655. res.Resp = resp
  656. if err != nil {
  657. rt--
  658. if rt == 0 {
  659. results <- &res
  660. break
  661. }
  662. continue
  663. }
  664. defer resp.Body.Close()
  665. fd, err := os.OpenFile(j.FilePath, os.O_WRONLY, 0660)
  666. if err != nil {
  667. res.err = err
  668. results <- &res
  669. break
  670. }
  671. fd.Seek(j.Chunk.OffSet, os.SEEK_SET)
  672. n, err := io.Copy(fd, LimitReadCloser(resp.Body, j.Chunk.Size))
  673. if n != j.Chunk.Size || err != nil {
  674. res.err = fmt.Errorf("io.Copy Failed, nread:%v, want:%v, err:%v", n, j.Chunk.Size, err)
  675. }
  676. fd.Close()
  677. results <- &res
  678. break
  679. }
  680. }
  681. }
  682. func DividePart(fileSize int64, last int) (int64, int64) {
  683. partSize := int64(last * 1024 * 1024)
  684. partNum := fileSize / partSize
  685. for partNum >= 10000 {
  686. partSize = partSize * 2
  687. partNum = fileSize / partSize
  688. }
  689. return partNum, partSize
  690. }
  691. func SplitFileIntoChunks(filePath string, partSize int64) (int64, []Chunk, int, error) {
  692. if filePath == "" {
  693. return 0, nil, 0, errors.New("filePath invalid")
  694. }
  695. file, err := os.Open(filePath)
  696. if err != nil {
  697. return 0, nil, 0, err
  698. }
  699. defer file.Close()
  700. stat, err := file.Stat()
  701. if err != nil {
  702. return 0, nil, 0, err
  703. }
  704. var partNum int64
  705. if partSize > 0 {
  706. if partSize < 1024*1024 {
  707. return 0, nil, 0, errors.New("partSize>=1048576 is required")
  708. }
  709. partNum = stat.Size() / partSize
  710. if partNum >= 10000 {
  711. return 0, nil, 0, errors.New("Too many parts, out of 10000")
  712. }
  713. } else {
  714. partNum, partSize = DividePart(stat.Size(), 64)
  715. }
  716. var chunks []Chunk
  717. var chunk = Chunk{}
  718. for i := int64(0); i < partNum; i++ {
  719. chunk.Number = int(i + 1)
  720. chunk.OffSet = i * partSize
  721. chunk.Size = partSize
  722. chunks = append(chunks, chunk)
  723. }
  724. if stat.Size()%partSize > 0 {
  725. chunk.Number = len(chunks) + 1
  726. chunk.OffSet = int64(len(chunks)) * partSize
  727. chunk.Size = stat.Size() % partSize
  728. chunks = append(chunks, chunk)
  729. partNum++
  730. }
  731. return int64(stat.Size()), chunks, int(partNum), nil
  732. }
  733. func (s *ObjectService) getResumableUploadID(ctx context.Context, name string) (string, error) {
  734. opt := &ObjectListUploadsOptions{
  735. Prefix: name,
  736. EncodingType: "url",
  737. }
  738. res, _, err := s.ListUploads(ctx, opt)
  739. if err != nil {
  740. return "", err
  741. }
  742. if len(res.Upload) == 0 {
  743. return "", nil
  744. }
  745. last := len(res.Upload) - 1
  746. for last >= 0 {
  747. decodeKey, _ := decodeURIComponent(res.Upload[last].Key)
  748. if decodeKey == name {
  749. return decodeURIComponent(res.Upload[last].UploadID)
  750. }
  751. last = last - 1
  752. }
  753. return "", nil
  754. }
  755. func (s *ObjectService) checkUploadedParts(ctx context.Context, name, UploadID, filepath string, chunks []Chunk, partNum int) error {
  756. var uploadedParts []Object
  757. isTruncated := true
  758. opt := &ObjectListPartsOptions{
  759. EncodingType: "url",
  760. }
  761. for isTruncated {
  762. res, _, err := s.ListParts(ctx, name, UploadID, opt)
  763. if err != nil {
  764. return err
  765. }
  766. if len(res.Parts) > 0 {
  767. uploadedParts = append(uploadedParts, res.Parts...)
  768. }
  769. isTruncated = res.IsTruncated
  770. opt.PartNumberMarker = res.NextPartNumberMarker
  771. }
  772. fd, err := os.Open(filepath)
  773. if err != nil {
  774. return err
  775. }
  776. defer fd.Close()
  777. // 某个分块出错, 重置chunks
  778. ret := func(e error) error {
  779. for i, _ := range chunks {
  780. chunks[i].Done = false
  781. chunks[i].ETag = ""
  782. }
  783. return e
  784. }
  785. for _, part := range uploadedParts {
  786. partNumber := part.PartNumber
  787. if partNumber > partNum {
  788. return ret(errors.New("Part Number is not consistent"))
  789. }
  790. partNumber = partNumber - 1
  791. fd.Seek(chunks[partNumber].OffSet, os.SEEK_SET)
  792. bs, err := ioutil.ReadAll(io.LimitReader(fd, chunks[partNumber].Size))
  793. if err != nil {
  794. return ret(err)
  795. }
  796. localMD5 := fmt.Sprintf("\"%x\"", md5.Sum(bs))
  797. if localMD5 != part.ETag {
  798. return ret(errors.New(fmt.Sprintf("CheckSum Failed in Part[%d]", part.PartNumber)))
  799. }
  800. chunks[partNumber].Done = true
  801. chunks[partNumber].ETag = part.ETag
  802. }
  803. return nil
  804. }
  805. // MultiUpload/Upload 为高级upload接口,并发分块上传
  806. //
  807. // 当 partSize > 0 时,由调用者指定分块大小,否则由 SDK 自动切分,单位为MB
  808. // 由调用者指定分块大小时,请确认分块数量不超过10000
  809. //
  810. func (s *ObjectService) MultiUpload(ctx context.Context, name string, filepath string, opt *MultiUploadOptions) (*CompleteMultipartUploadResult, *Response, error) {
  811. return s.Upload(ctx, name, filepath, opt)
  812. }
  813. func (s *ObjectService) Upload(ctx context.Context, name string, filepath string, opt *MultiUploadOptions) (*CompleteMultipartUploadResult, *Response, error) {
  814. if opt == nil {
  815. opt = &MultiUploadOptions{}
  816. }
  817. var localcrc uint64
  818. // 1.Get the file chunk
  819. totalBytes, chunks, partNum, err := SplitFileIntoChunks(filepath, opt.PartSize*1024*1024)
  820. if err != nil {
  821. return nil, nil, err
  822. }
  823. // 校验
  824. if s.client.Conf.EnableCRC {
  825. fd, err := os.Open(filepath)
  826. if err != nil {
  827. return nil, nil, err
  828. }
  829. defer fd.Close()
  830. localcrc, err = calCRC64(fd)
  831. if err != nil {
  832. return nil, nil, err
  833. }
  834. }
  835. // filesize=0 , use simple upload
  836. if partNum == 0 || partNum == 1 {
  837. var opt0 *ObjectPutOptions
  838. if opt.OptIni != nil {
  839. opt0 = &ObjectPutOptions{
  840. opt.OptIni.ACLHeaderOptions,
  841. opt.OptIni.ObjectPutHeaderOptions,
  842. }
  843. }
  844. rsp, err := s.PutFromFile(ctx, name, filepath, opt0)
  845. if err != nil {
  846. return nil, rsp, err
  847. }
  848. result := &CompleteMultipartUploadResult{
  849. Location: fmt.Sprintf("%s/%s", s.client.BaseURL.BucketURL, name),
  850. Key: name,
  851. ETag: rsp.Header.Get("ETag"),
  852. }
  853. if rsp != nil && s.client.Conf.EnableCRC {
  854. scoscrc := rsp.Header.Get("x-cos-hash-crc64ecma")
  855. icoscrc, _ := strconv.ParseUint(scoscrc, 10, 64)
  856. if icoscrc != localcrc {
  857. return result, rsp, fmt.Errorf("verification failed, want:%v, return:%v", localcrc, icoscrc)
  858. }
  859. }
  860. return result, rsp, nil
  861. }
  862. var uploadID string
  863. resumableFlag := false
  864. if opt.CheckPoint {
  865. var err error
  866. uploadID, err = s.getResumableUploadID(ctx, name)
  867. if err == nil && uploadID != "" {
  868. err = s.checkUploadedParts(ctx, name, uploadID, filepath, chunks, partNum)
  869. resumableFlag = (err == nil)
  870. }
  871. }
  872. // 2.Init
  873. optini := opt.OptIni
  874. if !resumableFlag {
  875. res, _, err := s.InitiateMultipartUpload(ctx, name, optini)
  876. if err != nil {
  877. return nil, nil, err
  878. }
  879. uploadID = res.UploadID
  880. }
  881. var poolSize int
  882. if opt.ThreadPoolSize > 0 {
  883. poolSize = opt.ThreadPoolSize
  884. } else {
  885. // Default is one
  886. poolSize = 1
  887. }
  888. chjobs := make(chan *Jobs, 100)
  889. chresults := make(chan *Results, 10000)
  890. optcom := &CompleteMultipartUploadOptions{}
  891. // 3.Start worker
  892. for w := 1; w <= poolSize; w++ {
  893. go worker(s, chjobs, chresults)
  894. }
  895. // progress started event
  896. var listener ProgressListener
  897. var consumedBytes int64
  898. if opt.OptIni != nil {
  899. if opt.OptIni.ObjectPutHeaderOptions != nil {
  900. listener = opt.OptIni.Listener
  901. }
  902. optcom.XOptionHeader, _ = deliverInitOptions(opt.OptIni)
  903. }
  904. event := newProgressEvent(ProgressStartedEvent, 0, 0, totalBytes)
  905. progressCallback(listener, event)
  906. // 4.Push jobs
  907. go func() {
  908. for _, chunk := range chunks {
  909. if chunk.Done {
  910. continue
  911. }
  912. partOpt := &ObjectUploadPartOptions{}
  913. if optini != nil && optini.ObjectPutHeaderOptions != nil {
  914. partOpt.XCosSSECustomerAglo = optini.XCosSSECustomerAglo
  915. partOpt.XCosSSECustomerKey = optini.XCosSSECustomerKey
  916. partOpt.XCosSSECustomerKeyMD5 = optini.XCosSSECustomerKeyMD5
  917. partOpt.XCosTrafficLimit = optini.XCosTrafficLimit
  918. }
  919. job := &Jobs{
  920. Name: name,
  921. RetryTimes: 3,
  922. FilePath: filepath,
  923. UploadId: uploadID,
  924. Chunk: chunk,
  925. Opt: partOpt,
  926. }
  927. chjobs <- job
  928. }
  929. close(chjobs)
  930. }()
  931. // 5.Recv the resp etag to complete
  932. err = nil
  933. for i := 0; i < partNum; i++ {
  934. if chunks[i].Done {
  935. optcom.Parts = append(optcom.Parts, Object{
  936. PartNumber: chunks[i].Number, ETag: chunks[i].ETag},
  937. )
  938. if err == nil {
  939. consumedBytes += chunks[i].Size
  940. event = newProgressEvent(ProgressDataEvent, chunks[i].Size, consumedBytes, totalBytes)
  941. progressCallback(listener, event)
  942. }
  943. continue
  944. }
  945. res := <-chresults
  946. // Notice one part fail can not get the etag according.
  947. if res.Resp == nil || res.err != nil {
  948. // Some part already fail, can not to get the header inside.
  949. err = fmt.Errorf("UploadID %s, part %d failed to get resp content. error: %s", uploadID, res.PartNumber, res.err.Error())
  950. continue
  951. }
  952. // Notice one part fail can not get the etag according.
  953. etag := res.Resp.Header.Get("ETag")
  954. optcom.Parts = append(optcom.Parts, Object{
  955. PartNumber: res.PartNumber, ETag: etag},
  956. )
  957. if err == nil {
  958. consumedBytes += chunks[res.PartNumber-1].Size
  959. event = newProgressEvent(ProgressDataEvent, chunks[res.PartNumber-1].Size, consumedBytes, totalBytes)
  960. progressCallback(listener, event)
  961. }
  962. }
  963. close(chresults)
  964. if err != nil {
  965. event = newProgressEvent(ProgressFailedEvent, 0, consumedBytes, totalBytes, err)
  966. progressCallback(listener, event)
  967. return nil, nil, err
  968. }
  969. sort.Sort(ObjectList(optcom.Parts))
  970. event = newProgressEvent(ProgressCompletedEvent, 0, consumedBytes, totalBytes)
  971. progressCallback(listener, event)
  972. v, resp, err := s.CompleteMultipartUpload(context.Background(), name, uploadID, optcom)
  973. if err != nil {
  974. return v, resp, err
  975. }
  976. if resp != nil && s.client.Conf.EnableCRC {
  977. scoscrc := resp.Header.Get("x-cos-hash-crc64ecma")
  978. icoscrc, _ := strconv.ParseUint(scoscrc, 10, 64)
  979. if icoscrc != localcrc {
  980. return v, resp, fmt.Errorf("verification failed, want:%v, return:%v", localcrc, icoscrc)
  981. }
  982. }
  983. return v, resp, err
  984. }
  985. func SplitSizeIntoChunks(totalBytes int64, partSize int64) ([]Chunk, int, error) {
  986. var partNum int64
  987. if partSize > 0 {
  988. if partSize < 1024*1024 {
  989. return nil, 0, errors.New("partSize>=1048576 is required")
  990. }
  991. partNum = totalBytes / partSize
  992. if partNum >= 10000 {
  993. return nil, 0, errors.New("Too manry parts, out of 10000")
  994. }
  995. } else {
  996. partNum, partSize = DividePart(totalBytes, 64)
  997. }
  998. var chunks []Chunk
  999. var chunk = Chunk{}
  1000. for i := int64(0); i < partNum; i++ {
  1001. chunk.Number = int(i + 1)
  1002. chunk.OffSet = i * partSize
  1003. chunk.Size = partSize
  1004. chunks = append(chunks, chunk)
  1005. }
  1006. if totalBytes%partSize > 0 {
  1007. chunk.Number = len(chunks) + 1
  1008. chunk.OffSet = int64(len(chunks)) * partSize
  1009. chunk.Size = totalBytes % partSize
  1010. chunks = append(chunks, chunk)
  1011. partNum++
  1012. }
  1013. return chunks, int(partNum), nil
  1014. }
  1015. func (s *ObjectService) checkDownloadedParts(opt *MultiDownloadCPInfo, chfile string, chunks []Chunk) (*MultiDownloadCPInfo, bool) {
  1016. var defaultRes MultiDownloadCPInfo
  1017. defaultRes = *opt
  1018. fd, err := os.Open(chfile)
  1019. // checkpoint 文件不存在
  1020. if err != nil && os.IsNotExist(err) {
  1021. // 创建 checkpoint 文件
  1022. fd, _ = os.OpenFile(chfile, os.O_RDONLY|os.O_CREATE|os.O_TRUNC, 0660)
  1023. fd.Close()
  1024. return &defaultRes, false
  1025. }
  1026. if err != nil {
  1027. return &defaultRes, false
  1028. }
  1029. defer fd.Close()
  1030. var res MultiDownloadCPInfo
  1031. err = json.NewDecoder(fd).Decode(&res)
  1032. if err != nil {
  1033. return &defaultRes, false
  1034. }
  1035. // 与COS的文件比较
  1036. if res.CRC64 != opt.CRC64 || res.ETag != opt.ETag || res.Size != opt.Size || res.LastModified != opt.LastModified || len(res.DownloadedBlocks) == 0 {
  1037. return &defaultRes, false
  1038. }
  1039. // len(chunks) 大于1,否则为简单下载, chunks[0].Size为partSize
  1040. partSize := chunks[0].Size
  1041. for _, v := range res.DownloadedBlocks {
  1042. index := v.From / partSize
  1043. to := chunks[index].OffSet + chunks[index].Size - 1
  1044. if chunks[index].OffSet != v.From || to != v.To {
  1045. // 重置chunks
  1046. for i, _ := range chunks {
  1047. chunks[i].Done = false
  1048. }
  1049. return &defaultRes, false
  1050. }
  1051. chunks[index].Done = true
  1052. }
  1053. return &res, true
  1054. }
  1055. func (s *ObjectService) Download(ctx context.Context, name string, filepath string, opt *MultiDownloadOptions, id ...string) (*Response, error) {
  1056. // 参数校验
  1057. if opt == nil {
  1058. opt = &MultiDownloadOptions{}
  1059. }
  1060. if opt.Opt != nil && opt.Opt.Range != "" {
  1061. return nil, fmt.Errorf("Download doesn't support Range Options")
  1062. }
  1063. // 获取文件长度和CRC
  1064. var coscrc string
  1065. resp, err := s.Head(ctx, name, nil, id...)
  1066. if err != nil {
  1067. return resp, err
  1068. }
  1069. // 如果对象不存在x-cos-hash-crc64ecma,则跳过不做校验
  1070. coscrc = resp.Header.Get("x-cos-hash-crc64ecma")
  1071. strTotalBytes := resp.Header.Get("Content-Length")
  1072. totalBytes, err := strconv.ParseInt(strTotalBytes, 10, 64)
  1073. if err != nil {
  1074. return resp, err
  1075. }
  1076. // 切分
  1077. chunks, partNum, err := SplitSizeIntoChunks(totalBytes, opt.PartSize*1024*1024)
  1078. if err != nil {
  1079. return resp, err
  1080. }
  1081. // 直接下载到文件
  1082. if partNum == 0 || partNum == 1 {
  1083. rsp, err := s.GetToFile(ctx, name, filepath, opt.Opt, id...)
  1084. if err != nil {
  1085. return rsp, err
  1086. }
  1087. if coscrc != "" && s.client.Conf.EnableCRC {
  1088. icoscrc, _ := strconv.ParseUint(coscrc, 10, 64)
  1089. fd, err := os.Open(filepath)
  1090. if err != nil {
  1091. return rsp, err
  1092. }
  1093. defer fd.Close()
  1094. localcrc, err := calCRC64(fd)
  1095. if err != nil {
  1096. return rsp, err
  1097. }
  1098. if localcrc != icoscrc {
  1099. return rsp, fmt.Errorf("verification failed, want:%v, return:%v", icoscrc, localcrc)
  1100. }
  1101. }
  1102. return rsp, err
  1103. }
  1104. // 断点续载
  1105. var resumableFlag bool
  1106. var resumableInfo *MultiDownloadCPInfo
  1107. var cpfd *os.File
  1108. var cpfile string
  1109. if opt.CheckPoint {
  1110. cpInfo := &MultiDownloadCPInfo{
  1111. LastModified: resp.Header.Get("Last-Modified"),
  1112. ETag: resp.Header.Get("ETag"),
  1113. CRC64: coscrc,
  1114. Size: totalBytes,
  1115. }
  1116. cpfile = opt.CheckPointFile
  1117. if cpfile == "" {
  1118. cpfile = fmt.Sprintf("%s.cosresumabletask", filepath)
  1119. }
  1120. resumableInfo, resumableFlag = s.checkDownloadedParts(cpInfo, cpfile, chunks)
  1121. cpfd, err = os.OpenFile(cpfile, os.O_RDWR, 0660)
  1122. if err != nil {
  1123. return nil, fmt.Errorf("Open CheckPoint File[%v] Failed:%v", cpfile, err)
  1124. }
  1125. }
  1126. if !resumableFlag {
  1127. // 创建文件
  1128. nfile, err := os.OpenFile(filepath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660)
  1129. if err != nil {
  1130. if cpfd != nil {
  1131. cpfd.Close()
  1132. }
  1133. return resp, err
  1134. }
  1135. nfile.Close()
  1136. }
  1137. var poolSize int
  1138. if opt.ThreadPoolSize > 0 {
  1139. poolSize = opt.ThreadPoolSize
  1140. } else {
  1141. poolSize = 1
  1142. }
  1143. chjobs := make(chan *Jobs, 100)
  1144. chresults := make(chan *Results, 10000)
  1145. for w := 1; w <= poolSize; w++ {
  1146. go downloadWorker(s, chjobs, chresults)
  1147. }
  1148. go func() {
  1149. for _, chunk := range chunks {
  1150. if chunk.Done {
  1151. continue
  1152. }
  1153. var downOpt ObjectGetOptions
  1154. if opt.Opt != nil {
  1155. downOpt = *opt.Opt
  1156. downOpt.Listener = nil // listener need to set nil
  1157. }
  1158. job := &Jobs{
  1159. Name: name,
  1160. RetryTimes: 3,
  1161. FilePath: filepath,
  1162. Chunk: chunk,
  1163. DownOpt: &downOpt,
  1164. }
  1165. if len(id) > 0 {
  1166. job.VersionId = append(job.VersionId, id...)
  1167. }
  1168. chjobs <- job
  1169. }
  1170. close(chjobs)
  1171. }()
  1172. err = nil
  1173. for i := 0; i < partNum; i++ {
  1174. if chunks[i].Done {
  1175. continue
  1176. }
  1177. res := <-chresults
  1178. if res.Resp == nil || res.err != nil {
  1179. err = fmt.Errorf("part %d get resp Content. error: %s", res.PartNumber, res.err.Error())
  1180. continue
  1181. }
  1182. // Dump CheckPoint Info
  1183. if opt.CheckPoint {
  1184. cpfd.Truncate(0)
  1185. cpfd.Seek(0, os.SEEK_SET)
  1186. resumableInfo.DownloadedBlocks = append(resumableInfo.DownloadedBlocks, DownloadedBlock{
  1187. From: chunks[res.PartNumber-1].OffSet,
  1188. To: chunks[res.PartNumber-1].OffSet + chunks[res.PartNumber-1].Size - 1,
  1189. })
  1190. json.NewEncoder(cpfd).Encode(resumableInfo)
  1191. }
  1192. }
  1193. close(chresults)
  1194. if cpfd != nil {
  1195. cpfd.Close()
  1196. }
  1197. if err != nil {
  1198. return nil, err
  1199. }
  1200. // 下载成功,删除checkpoint文件
  1201. if opt.CheckPoint {
  1202. os.Remove(cpfile)
  1203. }
  1204. if coscrc != "" && s.client.Conf.EnableCRC {
  1205. icoscrc, _ := strconv.ParseUint(coscrc, 10, 64)
  1206. fd, err := os.Open(filepath)
  1207. if err != nil {
  1208. return resp, err
  1209. }
  1210. defer fd.Close()
  1211. localcrc, err := calCRC64(fd)
  1212. if err != nil {
  1213. return resp, err
  1214. }
  1215. if localcrc != icoscrc {
  1216. return resp, fmt.Errorf("verification failed, want:%v, return:%v", icoscrc, localcrc)
  1217. }
  1218. }
  1219. return resp, err
  1220. }
  1221. type ObjectPutTaggingOptions struct {
  1222. XMLName xml.Name `xml:"Tagging"`
  1223. TagSet []ObjectTaggingTag `xml:"TagSet>Tag,omitempty"`
  1224. }
  1225. type ObjectTaggingTag BucketTaggingTag
  1226. type ObjectGetTaggingResult ObjectPutTaggingOptions
  1227. func (s *ObjectService) PutTagging(ctx context.Context, name string, opt *ObjectPutTaggingOptions, id ...string) (*Response, error) {
  1228. var u string
  1229. if len(id) == 1 {
  1230. u = fmt.Sprintf("/%s?tagging&versionId=%s", encodeURIComponent(name), id[0])
  1231. } else if len(id) == 0 {
  1232. u = fmt.Sprintf("/%s?tagging", encodeURIComponent(name))
  1233. } else {
  1234. return nil, errors.New("wrong params")
  1235. }
  1236. sendOpt := &sendOptions{
  1237. baseURL: s.client.BaseURL.BucketURL,
  1238. uri: u,
  1239. method: http.MethodPut,
  1240. body: opt,
  1241. }
  1242. resp, err := s.client.send(ctx, sendOpt)
  1243. return resp, err
  1244. }
  1245. func (s *ObjectService) GetTagging(ctx context.Context, name string, id ...string) (*ObjectGetTaggingResult, *Response, error) {
  1246. var u string
  1247. if len(id) == 1 {
  1248. u = fmt.Sprintf("/%s?tagging&versionId=%s", encodeURIComponent(name), id[0])
  1249. } else if len(id) == 0 {
  1250. u = fmt.Sprintf("/%s?tagging", encodeURIComponent(name))
  1251. } else {
  1252. return nil, nil, errors.New("wrong params")
  1253. }
  1254. var res ObjectGetTaggingResult
  1255. sendOpt := &sendOptions{
  1256. baseURL: s.client.BaseURL.BucketURL,
  1257. uri: u,
  1258. method: http.MethodGet,
  1259. result: &res,
  1260. }
  1261. resp, err := s.client.send(ctx, sendOpt)
  1262. return &res, resp, err
  1263. }
  1264. func (s *ObjectService) DeleteTagging(ctx context.Context, name string, id ...string) (*Response, error) {
  1265. var u string
  1266. if len(id) == 1 {
  1267. u = fmt.Sprintf("/%s?tagging&versionId=%s", encodeURIComponent(name), id[0])
  1268. } else if len(id) == 0 {
  1269. u = fmt.Sprintf("/%s?tagging", encodeURIComponent(name))
  1270. } else {
  1271. return nil, errors.New("wrong params")
  1272. }
  1273. sendOpt := &sendOptions{
  1274. baseURL: s.client.BaseURL.BucketURL,
  1275. uri: u,
  1276. method: http.MethodDelete,
  1277. }
  1278. resp, err := s.client.send(ctx, sendOpt)
  1279. return resp, err
  1280. }