update download retry && bucket domain

bucket_domain.go

@@ -6,12 +6,16 @@ import (
 	"net/http"
 )
 
+type BucketDomainRule struct {
+	Status            string `xml:"Status,omitempty"`
+	Name              string `xml:"Name,omitempty"`
+	Type              string `xml:"Type,omitempty"`
+	ForcedReplacement string `xml:"ForcedReplacement,omitempty"`
+}
+
 type BucketPutDomainOptions struct {
-	XMLName           xml.Name `xml:"DomainConfiguration"`
-	Status            string   `xml:"DomainRule>Status"`
-	Name              string   `xml:"DomainRule>Name"`
-	Type              string   `xml:"DomainRule>Type"`
-	ForcedReplacement string   `xml:"DomainRule>ForcedReplacement,omitempty"`
+	XMLName xml.Name           `xml:"DomainConfiguration"`
+	Rules   []BucketDomainRule `xml:"DomainRule,omitempty"`
 }
 type BucketGetDomainResult BucketPutDomainOptions
 
@@ -37,3 +41,13 @@ func (s *BucketService) GetDomain(ctx context.Context) (*BucketGetDomainResult,
 	resp, err := s.client.doRetry(ctx, sendOpt)
 	return &res, resp, err
 }
+
+func (s *BucketService) DeleteDomain(ctx context.Context) (*Response, error) {
+	sendOpt := &sendOptions{
+		baseURL: s.client.BaseURL.BucketURL,
+		uri:     "/?domain",
+		method:  http.MethodDelete,
+	}
+	resp, err := s.client.doRetry(ctx, sendOpt)
+	return resp, err
+}

bucket_domain_test.go

@@ -41,11 +41,15 @@ func TestBucketService_GetDomain(t *testing.T) {
 	}
 
 	want := &BucketGetDomainResult{
-		XMLName:           xml.Name{Local: "DomainConfiguration"},
-		Status:            "ENABLED",
-		Name:              "www.abc.com",
-		Type:              "REST",
-		ForcedReplacement: "CNAME",
+		XMLName: xml.Name{Local: "DomainConfiguration"},
+		Rules: []BucketDomainRule{
+			{
+				Status:            "ENABLED",
+				Name:              "www.abc.com",
+				Type:              "REST",
+				ForcedReplacement: "CNAME",
+			},
+		},
 	}
 
 	if !reflect.DeepEqual(res, want) {
@@ -58,11 +62,15 @@ func TestBucketService_PutDomain(t *testing.T) {
 	defer teardown()
 
 	opt := &BucketPutDomainOptions{
-		XMLName:           xml.Name{Local: "DomainConfiguration"},
-		Status:            "ENABLED",
-		Name:              "www.abc.com",
-		Type:              "REST",
-		ForcedReplacement: "CNAME",
+		XMLName: xml.Name{Local: "DomainConfiguration"},
+		Rules: []BucketDomainRule{
+			{
+				Status:            "ENABLED",
+				Name:              "www.abc.com",
+				Type:              "REST",
+				ForcedReplacement: "CNAME",
+			},
+		},
 	}
 
 	rt := 0
@@ -95,35 +103,18 @@ func TestBucketService_DeleteDomain(t *testing.T) {
 	setup()
 	defer teardown()
 
-	opt := &BucketPutDomainOptions{}
-
-	rt := 0
 	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
-		testMethod(t, r, http.MethodPut)
+		testMethod(t, r, http.MethodDelete)
 		vs := values{
 			"domain": "",
 		}
 		testFormValues(t, r, vs)
-
-		rt++
-		if rt < 3 {
-			w.WriteHeader(http.StatusGatewayTimeout)
-			return
-		}
-		body := new(BucketPutDomainOptions)
-		xml.NewDecoder(r.Body).Decode(body)
-		want := opt
-		want.XMLName = xml.Name{Local: "DomainConfiguration"}
-		if !reflect.DeepEqual(body, want) {
-			t.Errorf("Bucket.PutDomain request\n body: %+v\n, want %+v\n", body, want)
-		}
-
 		w.WriteHeader(http.StatusNoContent)
 	})
 
-	_, err := client.Bucket.PutDomain(context.Background(), opt)
+	_, err := client.Bucket.DeleteDomain(context.Background())
 	if err != nil {
-		t.Fatalf("Bucket.PutDomain returned error: %v", err)
+		t.Fatalf("Bucket.DeleteDomain returned error: %v", err)
 	}
 
 }

example/bucket/domain.go

@@ -0,0 +1,71 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/url"
+	"os"
+
+	"github.com/tencentyun/cos-go-sdk-v5"
+	"github.com/tencentyun/cos-go-sdk-v5/debug"
+)
+
+func log_status(err error) {
+	if err == nil {
+		return
+	}
+	if cos.IsNotFoundError(err) {
+		// WARN
+		fmt.Println("WARN: Resource is not existed")
+	} else if e, ok := cos.IsCOSError(err); ok {
+		fmt.Printf("ERROR: Code: %v\n", e.Code)
+		fmt.Printf("ERROR: Message: %v\n", e.Message)
+		fmt.Printf("ERROR: Resource: %v\n", e.Resource)
+		fmt.Printf("ERROR: RequestId: %v\n", e.RequestID)
+		// ERROR
+	} else {
+		fmt.Printf("ERROR: %v\n", err)
+		// ERROR
+	}
+}
+
+func main() {
+	u, _ := url.Parse("https://test-1259654469.cos.ap-guangzhou.myqcloud.com")
+	b := &cos.BaseURL{
+		BucketURL: u,
+	}
+	c := cos.NewClient(b, &http.Client{
+		Transport: &cos.AuthorizationTransport{
+			SecretID:  os.Getenv("COS_SECRETID"),
+			SecretKey: os.Getenv("COS_SECRETKEY"),
+			Transport: &debug.DebugRequestTransport{
+				RequestHeader:  true,
+				RequestBody:    true,
+				ResponseHeader: true,
+				ResponseBody:   true,
+			},
+		},
+	})
+
+	opt := &cos.BucketPutDomainOptions{
+		Rules: []cos.BucketDomainRule{
+			{
+				Status:            "ENABLED",
+				Name:              "www.qq.com",
+				Type:              "REST",
+				ForcedReplacement: "CNAME",
+			},
+		},
+	}
+
+	_, err := c.Bucket.PutDomain(context.Background(), opt)
+	log_status(err)
+
+	res, _, err := c.Bucket.GetDomain(context.Background())
+	log_status(err)
+	fmt.Printf("%+v\n", res)
+
+	_, err = c.Bucket.DeleteDomain(context.Background())
+	log_status(err)
+}

example/bucket/getDomain.go

@@ -1,35 +0,0 @@
-package main
-
-import (
-	"context"
-	"net/http"
-	"net/url"
-	"os"
-
-	"github.com/tencentyun/cos-go-sdk-v5"
-	"github.com/tencentyun/cos-go-sdk-v5/debug"
-)
-
-func main() {
-	u, _ := url.Parse("https://test-1259654469.cos.ap-guangzhou.myqcloud.com")
-	b := &cos.BaseURL{
-		BucketURL: u,
-	}
-	c := cos.NewClient(b, &http.Client{
-		Transport: &cos.AuthorizationTransport{
-			SecretID:  os.Getenv("COS_SECRETID"),
-			SecretKey: os.Getenv("COS_SECRETKEY"),
-			Transport: &debug.DebugRequestTransport{
-				RequestHeader:  true,
-				RequestBody:    true,
-				ResponseHeader: true,
-				ResponseBody:   true,
-			},
-		},
-	})
-
-	_, _, err := c.Bucket.GetDomain(context.Background())
-	if err != nil {
-		panic(err)
-	}
-}

example/bucket/putDomain.go

@@ -1,42 +0,0 @@
-package main
-
-import (
-	"context"
-	"net/http"
-	"net/url"
-	"os"
-
-	"github.com/tencentyun/cos-go-sdk-v5"
-	"github.com/tencentyun/cos-go-sdk-v5/debug"
-)
-
-func main() {
-	u, _ := url.Parse("https://test-1259654469.cos.ap-guangzhou.myqcloud.com")
-	b := &cos.BaseURL{
-		BucketURL: u,
-	}
-	c := cos.NewClient(b, &http.Client{
-		Transport: &cos.AuthorizationTransport{
-			SecretID:  os.Getenv("COS_SECRETID"),
-			SecretKey: os.Getenv("COS_SECRETKEY"),
-			Transport: &debug.DebugRequestTransport{
-				RequestHeader:  true,
-				RequestBody:    true,
-				ResponseHeader: true,
-				ResponseBody:   true,
-			},
-		},
-	})
-
-	opt := &cos.BucketPutDomainOptions{
-		Status:            "ENABLED",
-		Name:              "www.abc.com",
-		Type:              "REST",
-		ForcedReplacement: "CNAME",
-	}
-
-	_, err := c.Bucket.PutDomain(context.Background(), opt)
-	if err != nil {
-		panic(err)
-	}
-}

object.go

@@ -757,16 +757,12 @@ func downloadWorker(ctx context.Context, s *ObjectService, jobs <-chan *Jobs, re
 			res.err = err
 			res.Resp = resp
 			if err != nil {
-				rt--
-				if rt == 0 {
-					results <- &res
-					break
-				}
-				continue
+				results <- &res
+				break
 			}
-			defer resp.Body.Close()
 			fd, err := os.OpenFile(j.FilePath, os.O_WRONLY, 0660)
 			if err != nil {
+				resp.Body.Close()
 				res.err = err
 				results <- &res
 				break
@@ -774,9 +770,18 @@ func downloadWorker(ctx context.Context, s *ObjectService, jobs <-chan *Jobs, re
 			fd.Seek(j.Chunk.OffSet, os.SEEK_SET)
 			n, err := io.Copy(fd, LimitReadCloser(resp.Body, j.Chunk.Size))
 			if n != j.Chunk.Size || err != nil {
-				res.err = fmt.Errorf("io.Copy Failed, nread:%v, want:%v, err:%v", n, j.Chunk.Size, err)
+				fd.Close()
+				resp.Body.Close()
+				rt--
+				if rt == 0 {
+					res.err = fmt.Errorf("io.Copy Failed, nread:%v, want:%v, err:%v", n, j.Chunk.Size, err)
+					results <- &res
+					break
+				}
+				continue
 			}
 			fd.Close()
+			resp.Body.Close()
 			results <- &res
 			break
 		}
@@ -818,7 +823,7 @@ func SplitFileIntoChunks(filePath string, partSize int64) (int64, []Chunk, int,
 			return 0, nil, 0, errors.New("Too many parts, out of 10000")
 		}
 	} else {
-		partNum, partSize = DividePart(stat.Size(), 64)
+		partNum, partSize = DividePart(stat.Size(), 16)
 	}
 
 	var chunks []Chunk
@@ -1119,7 +1124,7 @@ func SplitSizeIntoChunks(totalBytes int64, partSize int64) ([]Chunk, int, error)
 			return nil, 0, errors.New("Too manry parts, out of 10000")
 		}
 	} else {
-		partNum, partSize = DividePart(totalBytes, 64)
+		partNum, partSize = DividePart(totalBytes, 16)
 	}
 
 	var chunks []Chunk
@@ -1293,7 +1298,7 @@ func (s *ObjectService) Download(ctx context.Context, name string, filepath stri
 			}
 			job := &Jobs{
 				Name:       name,
-				RetryTimes: 1,
+				RetryTimes: 3,
 				FilePath:   filepath,
 				Chunk:      chunk,
 				DownOpt:    &downOpt,

object_test.go

@@ -796,7 +796,7 @@ func TestObjectService_Download(t *testing.T) {
 	}
 	defer os.Remove(filePath)
 	// 源文件内容
-	totalBytes := int64(1024*1024*9 + 123)
+	totalBytes := int64(1024*1024 + 1230) //1024*1024*9 + 123)
 	b := make([]byte, totalBytes)
 	_, err = rand.Read(b)
 	newfile.Write(b)
@@ -817,20 +817,21 @@ func TestObjectService_Download(t *testing.T) {
 		start, _ := strconv.ParseInt(slice2[0], 10, 64)
 		end, _ := strconv.ParseInt(slice2[1], 10, 64)
 		if retryMap[start] == 0 {
-			// 重试校验1
+			// SDK 内部重试
 			retryMap[start]++
 			w.WriteHeader(http.StatusGatewayTimeout)
 		} else if retryMap[start] == 1 {
-			// 重试检验2
+			// SDK Download 做重试
 			retryMap[start]++
 			io.Copy(w, bytes.NewBuffer(b[start:end]))
 		} else if retryMap[start] == 2 {
-			// 重试检验3
+			// SDK Download 做重试
 			retryMap[start]++
-			st := math_rand.Int63n(totalBytes - 1024*1024)
-			et := st + end - start
+			st := start
+			et := st + math_rand.Int63n(1024)
 			io.Copy(w, bytes.NewBuffer(b[st:et+1]))
 		} else {
+			// SDK Download 成功
 			io.Copy(w, bytes.NewBuffer(b[start:end+1]))
 		}
 	})
@@ -842,18 +843,15 @@ func TestObjectService_Download(t *testing.T) {
 	downPath := "down.file" + time.Now().Format(time.RFC3339)
 	defer os.Remove(downPath)
 	_, err = client.Object.Download(context.Background(), "test.go.download", downPath, opt)
-	if err == nil {
-		// 长度不一致 Failed
-		t.Fatalf("Object.Upload returned error: %v", err)
-	}
-	_, err = client.Object.Download(context.Background(), "test.go.download", downPath, opt)
-	if err == nil {
-		// CRC不一致
+	if err != nil {
+		t.Fatalf("Object.Upload returned error: %v", err)
+	}
+	_, err = client.Object.Download(context.Background(), "test.go.download", downPath, opt)
+	if err != nil {
 		t.Fatalf("Object.Upload returned error: %v", err)
 	}
 	_, err = client.Object.Download(context.Background(), "test.go.download", downPath, opt)
 	if err != nil {
-		// 正确
 		t.Fatalf("Object.Upload returned error: %v", err)
 	}
 }
@@ -895,7 +893,7 @@ func TestObjectService_DownloadWithCheckPoint(t *testing.T) {
 			if oddok {
 				io.Copy(w, bytes.NewBuffer(b[start:end+1]))
 			} else {
-				// 数据校验失败, Download不会做重试
+				// 数据校验失败, Download做3次重试
 				io.Copy(w, bytes.NewBuffer(b[start:end]))
 			}
 			oddcount++
@@ -934,7 +932,7 @@ func TestObjectService_DownloadWithCheckPoint(t *testing.T) {
 	}
 	fd.Close()
 
-	if oddcount != 5 || evencount != 5 {
+	if oddcount != 15 || evencount != 5 {
 		t.Fatalf("Object.Download failed, odd:%v, even:%v", oddcount, evencount)
 	}
 	// 设置奇数块OK
@@ -944,7 +942,7 @@ func TestObjectService_DownloadWithCheckPoint(t *testing.T) {
 		// 下载成功
 		t.Fatalf("Object.Download returned error: %v", err)
 	}
-	if oddcount != 10 || evencount != 5 {
+	if oddcount != 20 || evencount != 5 {
 		t.Fatalf("Object.Download failed, odd:%v, even:%v", oddcount, evencount)
 	}
 }