From c323f1b90cd97b98708216a72c666d89bdbcad8b Mon Sep 17 00:00:00 2001 From: toranger Date: Thu, 31 Jan 2019 10:27:19 +0800 Subject: [PATCH 1/8] Add ci test with the testify and travis, test the working --- .travis.yml | 40 +-- Makefile | 4 +- cos.go | 1 + costesting/ci_test.go | 449 ++++++++++++++++++++++++++++++ example/object/get.go | 25 +- example/object/put.go | 9 + example/test.sh | 2 +- object.go | 35 +++ object_part.go | 16 ++ vendor/github.com/google/go-querystring | 1 + vendor/github.com/mozillazg/go-httpheader | 1 + vendor/github.com/stretchr/testify | 1 + 12 files changed, 562 insertions(+), 22 deletions(-) create mode 100644 costesting/ci_test.go create mode 160000 vendor/github.com/google/go-querystring create mode 160000 vendor/github.com/mozillazg/go-httpheader create mode 160000 vendor/github.com/stretchr/testify diff --git a/.travis.yml b/.travis.yml index 05703c8..39bf574 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,27 +1,29 @@ language: go go: - - 1.6 - - 1.7 - - 1.8 - - 1.9 - - master - +- '1.6' +- '1.7' +- '1.8' +- '1.9' +- 1.10.x +- master sudo: false - before_install: - - go get github.com/mattn/goveralls - +- go get github.com/mattn/goveralls install: - - go get - - go build - +- go get +- go build script: - - make test - - go test -coverprofile=cover.out github.com/tencentyun/cos-go-sdk-v5 - - $HOME/gopath/bin/goveralls -service=travis-ci -coverprofile=cover.out - +- make test +- make ci-test +- go test -coverprofile=cover.out github.com/tencentyun/cos-go-sdk-v5 +- "$HOME/gopath/bin/goveralls -service=travis-ci -coverprofile=cover.out" matrix: allow_failures: - - go: 1.6 - - go: 1.7 - - go: master + - go: 1.6 + - go: 1.7 + - go: master +env: + global: + - secure: XXB/cFVnJcAzhOZ2/zplwjhhhireQQGGRbNscPgQ0kpUQCyPZ6oIHvJMafuP4TVTJHEdMiaDxm0HNvgARuopXVaQNmK2UZj6xw40Ud7OT7ZUnw88xkQkXOI5GwG8oz9LqxIXUSItHegKXRLW0e1PoBdjZNv6lxGFAtuOcl9ekAg/q2lGIIQFefz6NK7gCmGYULKe+4J15VFldoYNM0JesxxxArTvtv8+k+U53oUwy9dex6z5oIA1zGIeKLcOD2xXgbjid/Ett3t0B2w3GfJWoM9rGV0eHgveOAUGe5tQkMKvl5LK1hj+93ZmU0MAG7x7t9jYKrFPqU/eDNJRMb4Ro6L7lIXVEKaBUkLx28PnwFQ5D043GBVtQGqYNcldZXIfbyYEHQZlD/BWFOt5YqTpGg+7Wm4NC3Yffqsurzk54juT7FftzVy0A8MFkqO+c5RHrOSUlm01pWXkGLHgZhUP5gEZEuUaoluSQTZksmAUJZ7F8DxwpE4SYBqfN27PZ87rWDNyOqNv1w1trzwx2IfdHHA+vfCZ7UM5e85gxFWUO2tJCUai2q21v3gBrcAgBOb6BwVzbWAorM2zY20f0l21XxOWMakA+r4JJA3s3EmcczcQeeL6pkFIAh+qKdFEPuyQTjH1mGpPzYFNbWtvPXijQo5PqyGrKL8W1t3ovwXMXoE= + - secure: bep0PPD/oYW5zY0QpeeC+WgFIya5DNRVmR92MO+e5BdFlSJPhstoG8bRh91EeftzC/Hyd3PUEIglPqTgZPxwysqW/81plsU95wV3qJi9gPi7+ZtYXH4xZTnaqgZsTr7jsKSVoKHSu7XqCtbSytW8YMN9wRWzG19/9hX2Z79Q6yNy5l9856Oyj1E2IXDjdZLPsWDhnZ8Vvk1wAVy2fc2esqKzHAZwm8n9vee2yR8vz7GXUszzpKvn4R43eNzdlFEHCmN0ANmxLJZmnYDpZHHfNf4slts+0S6I7awFXppuXUDaJPBRCia4XoFeSw+01IW1Vi0kAwvGLhxjJCWc4M/4ZU0byXDT11tDFvWa19NmnbYiizWiXNVecn1oNWYJqIKe7TTAMAtHSXAPmLX0rXuXKzwM09W6yrLFufCxyix9IOnenEbe9WwSdBbhmeLF3Wu/uVGkDog/FsXJM75sk956vV9UKh9zF4B9/NR8szJMF7shEs0Fbru5UUWheqg4AadPl3dhAWuj2+6NANa1LpH3JVD3II9dlXeMmMvsSwDvrYUaX/S8tf6JwZG0zCJK0TYp05rjxH+NIzWaMUTY7+HwYqqK3pOW3San0SlZiMq8N7GSnKUZ7WRQXYSB4gXHrg+mWyeVC7XnqiRtCwVi+LtPMu+YUbg7dwVi0vtKjYZYIUY= + - secure: Ob28vrOuHMKNKEtChkWbsaVv2SwLhcxXMnvGe4XN+y3mFvdhYnwpt6NdgThF8OCZ0761tvTRmvALfiZnO0uORjTtoHKkVPrnVIxlCcode0NVJZNHGn2fqjemdLKCnSeX7hm+9zeLpCnIvC+Sp3iZ3t2AH4AzgFx6nirWO3HwT5l9rNL9Q1CfwlOpNJJ36r9JTHwQnXmOfOmszUNoZ3rtiFXJ8dCi+BgY0lsiIRSiDkAH7KAPf86REM+ww81AaXG4/RuYx1Vj5zQCtZN7XEOViSXEbqqb8SrIFOccDu5FV12djg+4QS7FSjLVGrdIUcn4oI6pS24Et3oXf8xFx6JLYyGGhgZ2BsyJEx5vLQvkTWnMTrwZVRtCQ+g6lMUQpJhL2rBrmVBUqBFb5IH69O7corQm53n5qLM8IiosAQLfbOtML/1PyEpKCG2aOx1377Fx2yzxXW3ucP1PBqCzli0oCM2T52LfiNvZTzkIU6XJebBnzkZXepzOIFSur86kxgvQFElw9ro2X6XXPKU5S25xVaUSvaN1kmqLSkToJ9S1rmDYXnJR4aH0R2GcLw+EkMHFJJoAjnRHxrB4/1vOJbzmfS+qy6ShRhUMSD8gk4YJ6Y7o9h7oekuWOEn+XGhl29U9T5OApzHfoPEGZwLnpHxAiKJtQtv/TNhBIOFCjigsF7U= diff --git a/Makefile b/Makefile index 31779c3..e3b7aa3 100644 --- a/Makefile +++ b/Makefile @@ -17,4 +17,6 @@ lint: .PHONY: example example: - cd examples && sh test.sh + cd example && sh test.sh +ci-test: + cd costesting && go test -v diff --git a/cos.go b/cos.go index fe35741..bb82983 100644 --- a/cos.go +++ b/cos.go @@ -326,6 +326,7 @@ type ACLHeaderOptions struct { type ACLGrantee struct { Type string `xml:"type,attr"` UIN string `xml:"uin,omitempty"` + URI string `xml:"URI,omitempty"` ID string `xml:",omitempty"` DisplayName string `xml:",omitempty"` SubAccount string `xml:"Subaccount,omitempty"` diff --git a/costesting/ci_test.go b/costesting/ci_test.go new file mode 100644 index 0000000..e2ecdda --- /dev/null +++ b/costesting/ci_test.go @@ -0,0 +1,449 @@ +package cos + +// Basic imports +import ( + "context" + "fmt" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + "github.com/tencentyun/cos-go-sdk-v5" + "github.com/tencentyun/cos-go-sdk-v5/debug" + "io/ioutil" + "math/rand" + "net/http" + "net/url" + "os" + "strings" + "testing" + "time" +) + +// Define the suite, and absorb the built-in basic suite +// functionality from testify - including a T() method which +// returns the current testing context +type CosTestSuite struct { + suite.Suite + VariableThatShouldStartAtFive int + + // CI client + Client *cos.Client + + // Copy source client + CClient *cos.Client + + Region string + + Bucket string + + Appid string + + // test_object + TestObject string + + // special_file_name + SepFileName string +} + +func (s *CosTestSuite) SetupSuite() { + fmt.Println("Set up test") + // init + s.TestObject = "test.txt" + s.SepFileName = "中文" + "→↓←→↖↗↙↘! \"#$%&'()*+,-./0123456789:;<=>@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~" + + // CI client for test interface + // URL like this http://test-1253846586.cos.ap-guangzhou.myqcloud.com + u := os.Getenv("COS_BUCKET_URL") + + // Get the region + iu, _ := url.Parse(u) + p := strings.Split(iu.Host, ".") + assert.Equal(s.T(), 5, len(p), "Bucket host is not right") + s.Region = p[2] + + // Bucket name + pp := strings.Split(p[0], "-") + s.Bucket = pp[0] + s.Appid = pp[1] + + ib := &cos.BaseURL{BucketURL: iu} + s.Client = cos.NewClient(ib, &http.Client{ + Transport: &cos.AuthorizationTransport{ + SecretID: os.Getenv("COS_SECRETID"), + SecretKey: os.Getenv("COS_SECRETKEY"), + Transport: &debug.DebugRequestTransport{ + // RequestHeader: true, + // RequestBody: true, + // ResponseHeader: true, + // ResponseBody: true, + }, + }, + }) + + opt := &cos.BucketPutOptions{ + XCosACL: "public-read", + } + r, err := s.Client.Bucket.Put(context.Background(), opt) + if err != nil && r.StatusCode == 409 { + fmt.Println("BucketAlreadyOwnedByYou") + } else if err != nil { + assert.Nil(s.T(), err, "PutBucket Failed") + } +} + +// Begin of api test + +// Service API +func (s *CosTestSuite) TestGetService() { + _, _, err := s.Client.Service.Get(context.Background()) + assert.Nil(s.T(), err, "GetService Failed") +} + +// Bucket API +func (s *CosTestSuite) TestPutHeadDeleteBucket() { + u := "http://gosdkbuckettest-" + s.Appid + ".cos.ap-beijing-1.myqcloud.com" + iu, _ := url.Parse(u) + ib := &cos.BaseURL{BucketURL: iu} + client := cos.NewClient(ib, &http.Client{ + Transport: &cos.AuthorizationTransport{ + SecretID: os.Getenv("COS_SECRETID"), + SecretKey: os.Getenv("COS_SECRETKEY"), + }, + }) + r, err := client.Bucket.Put(context.Background(), nil) + if err != nil && r.StatusCode == 409 { + fmt.Println("BucketAlreadyOwnedByYou") + } else if err != nil { + assert.Nil(s.T(), err, "PutBucket Failed") + } + + time.Sleep(3 * time.Second) + _, err = client.Bucket.Head(context.Background()) + assert.Nil(s.T(), err, "HeadBucket Failed") + + _, err = client.Bucket.Delete(context.Background()) + assert.Nil(s.T(), err, "DeleteBucket Failed") + +} + +func (s *CosTestSuite) TestPutBucketACLIllegal() { + opt := &cos.BucketPutACLOptions{ + Header: &cos.ACLHeaderOptions{ + XCosACL: "public-read-writ", + }, + } + _, err := s.Client.Bucket.PutACL(context.Background(), opt) + assert.NotNil(s.T(), err, "PutBucketACL illegal Failed") +} + +func (s *CosTestSuite) TestPutGetBucketACLNormal() { + // with header + opt := &cos.BucketPutACLOptions{ + Header: &cos.ACLHeaderOptions{ + XCosACL: "private", + }, + } + _, err := s.Client.Bucket.PutACL(context.Background(), opt) + assert.Nil(s.T(), err, "PutBucketACL normal Failed") + + v, _, err := s.Client.Bucket.GetACL(context.Background()) + assert.Nil(s.T(), err, "GetBucketACL normal Failed") + assert.Equal(s.T(), 1, len(v.AccessControlList), "GetBucketACL normal Failed, must be private") + +} + +func (s *CosTestSuite) TestGetBucket() { + opt := &cos.BucketGetOptions{ + Prefix: "中文", + MaxKeys: 3, + } + _, _, err := s.Client.Bucket.Get(context.Background(), opt) + assert.Nil(s.T(), err, "GetBucket Failed") +} + +func (s *CosTestSuite) TestGetBucketLocation() { + v, _, err := s.Client.Bucket.GetLocation(context.Background()) + assert.Nil(s.T(), err, "GetLocation Failed") + assert.Equal(s.T(), s.Region, v.Location, "GetLocation wrong region") +} + +func (s *CosTestSuite) TestPutGetDeleteCORS() { + opt := &cos.BucketPutCORSOptions{ + Rules: []cos.BucketCORSRule{ + { + AllowedOrigins: []string{"http://www.qq.com"}, + AllowedMethods: []string{"PUT", "GET"}, + AllowedHeaders: []string{"x-cos-meta-test", "x-cos-xx"}, + MaxAgeSeconds: 500, + ExposeHeaders: []string{"x-cos-meta-test1"}, + }, + }, + } + _, err := s.Client.Bucket.PutCORS(context.Background(), opt) + assert.Nil(s.T(), err, "PutBucketCORS Failed") + + v, _, err := s.Client.Bucket.GetCORS(context.Background()) + assert.Nil(s.T(), err, "GetBucketCORS Failed") + assert.Equal(s.T(), 1, len(v.Rules), "GetBucketCORS wrong number rules") +} + +func (s *CosTestSuite) TestPutGetDeleteLifeCycle() { + lc := &cos.BucketPutLifecycleOptions{ + Rules: []cos.BucketLifecycleRule{ + { + ID: "1234", + Filter: &cos.BucketLifecycleFilter{Prefix: "test"}, + Status: "Enabled", + Transition: &cos.BucketLifecycleTransition{ + Days: 10, + StorageClass: "Standard", + }, + }, + }, + } + _, err := s.Client.Bucket.PutLifecycle(context.Background(), lc) + assert.Nil(s.T(), err, "PutBucketLifecycle Failed") + _, _, err = s.Client.Bucket.GetLifecycle(context.Background()) + assert.Nil(s.T(), err, "GetBucketLifecycle Failed") + _, err = s.Client.Bucket.DeleteLifecycle(context.Background()) + assert.Nil(s.T(), err, "DeleteBucketLifecycle Failed") +} + +func (s *CosTestSuite) TestListMultipartUploads() { + // Create new upload + name := "test_multipart.txt" + v, _, err := s.Client.Object.InitiateMultipartUpload(context.Background(), name, nil) + assert.Nil(s.T(), err, "InitiateMultipartUpload Failed") + id := v.UploadID + + // List + r, _, err := s.Client.Bucket.ListMultipartUploads(context.Background(), nil) + assert.Nil(s.T(), err, "ListMultipartUploads Failed") + assert.Equal(s.T(), 1, len(r.Uploads), "ListMultipartUploads wrong number uploads") + for _, p := range r.Uploads { + assert.Equal(s.T(), name, p.Key, "ListMultipartUploads wrong key") + assert.Equal(s.T(), id, p.UploadID, "ListMultipartUploads wrong uploadid") + } + + // Abort + _, err = s.Client.Object.AbortMultipartUpload(context.Background(), name, id) + assert.Nil(s.T(), err, "AbortMultipartUpload Failed") +} + +// Object API +func (s *CosTestSuite) TestPutHeadGetDeleteObject_10MB() { + name := "test/objectPut.go" + b := make([]byte, 1024*1024*10) + _, err := rand.Read(b) + content := fmt.Sprintf("%X", b) + f := strings.NewReader(content) + + _, err = s.Client.Object.Put(context.Background(), name, f, nil) + assert.Nil(s.T(), err, "PutObject Failed") + + _, err = s.Client.Object.Head(context.Background(), name, nil) + assert.Nil(s.T(), err, "HeadObject Failed") + + _, err = s.Client.Object.Delete(context.Background(), name) + assert.Nil(s.T(), err, "DeleteObject Failed") +} + +func (s *CosTestSuite) TestPutGetDeleteObjectByFile_10MB() { + // Create tmp file + filePath := "tmpfile" + newfile, err := os.Create(filePath) + assert.Nil(s.T(), err, "create tmp file Failed") + defer newfile.Close() + + name := "test/objectPutByFile.go" + b := make([]byte, 1024*1024*10) + _, err = rand.Read(b) + + newfile.Write(b) + _, err = s.Client.Object.PutFromFile(context.Background(), name, filePath, nil) + assert.Nil(s.T(), err, "PutObject Failed") + + // Over write tmp file + _, err = s.Client.Object.GetToFile(context.Background(), name, filePath, nil) + assert.Nil(s.T(), err, "HeadObject Failed") + + _, err = s.Client.Object.Delete(context.Background(), name) + assert.Nil(s.T(), err, "DeleteObject Failed") + + // remove the local tmp file + err = os.Remove(filePath) + assert.Nil(s.T(), err, "remove local file Failed") +} + +func (s *CosTestSuite) TestPutGetDeleteObjectSpecialName() { + f := strings.NewReader("test") + _, err := s.Client.Object.Put(context.Background(), s.SepFileName, f, nil) + assert.Nil(s.T(), err, "PutObject Failed") + + resp, err := s.Client.Object.Get(context.Background(), s.SepFileName, nil) + assert.Nil(s.T(), err, "GetObject Failed") + defer resp.Body.Close() + bs, _ := ioutil.ReadAll(resp.Body) + assert.Equal(s.T(), "test", string(bs), "GetObject failed content wrong") + + _, err = s.Client.Object.Delete(context.Background(), s.SepFileName) + assert.Nil(s.T(), err, "DeleteObject Failed") +} + +func (s *CosTestSuite) TestPutObjectToNonExistBucket() { + u := "http://gosdknonexistbucket-" + s.Appid + ".cos." + s.Region + ".myqcloud.com" + iu, _ := url.Parse(u) + ib := &cos.BaseURL{BucketURL: iu} + client := cos.NewClient(ib, &http.Client{ + Transport: &cos.AuthorizationTransport{ + SecretID: os.Getenv("COS_SECRETID"), + SecretKey: os.Getenv("COS_SECRETKEY"), + }, + }) + name := "test/objectPut.go" + f := strings.NewReader("test") + r, err := client.Object.Put(context.Background(), name, f, nil) + assert.NotNil(s.T(), err, "PutObject ToNonExistBucket Failed") + assert.Equal(s.T(), 404, r.StatusCode, "PutObject ToNonExistBucket, not 404") +} + +func (s *CosTestSuite) TestPutGetObjectACL() { + name := "test/objectACL.go" + f := strings.NewReader("test") + _, err := s.Client.Object.Put(context.Background(), name, f, nil) + assert.Nil(s.T(), err, "PutObject Failed") + + // Put acl + opt := &cos.ObjectPutACLOptions{ + Header: &cos.ACLHeaderOptions{ + XCosACL: "public-read", + }, + } + _, err = s.Client.Object.PutACL(context.Background(), name, opt) + assert.Nil(s.T(), err, "PutObjectACL Failed") + v, _, err := s.Client.Object.GetACL(context.Background(), name) + assert.Nil(s.T(), err, "GetObjectACL Failed") + assert.Equal(s.T(), 2, len(v.AccessControlList), "GetLifecycle wrong number rules") + + _, err = s.Client.Object.Delete(context.Background(), name) + assert.Nil(s.T(), err, "DeleteObject Failed") +} + +func (s *CosTestSuite) TestCopyObject() { + u := "http://gosdkcopytest-" + s.Appid + ".cos.ap-beijing-1.myqcloud.com" + iu, _ := url.Parse(u) + ib := &cos.BaseURL{BucketURL: iu} + c := cos.NewClient(ib, &http.Client{ + Transport: &cos.AuthorizationTransport{ + SecretID: os.Getenv("COS_SECRETID"), + SecretKey: os.Getenv("COS_SECRETKEY"), + }, + }) + + opt := &cos.BucketPutOptions{ + XCosACL: "public-read", + } + + // Notice in intranet the bucket host sometimes has i/o timeout problem + r, err := c.Bucket.Put(context.Background(), opt) + if err != nil && r.StatusCode == 409 { + fmt.Println("BucketAlreadyOwnedByYou") + } else if err != nil { + assert.Nil(s.T(), err, "PutBucket Failed") + } + + source := "test/objectMove1.go" + expected := "test" + f := strings.NewReader(expected) + + _, err = c.Object.Put(context.Background(), source, f, nil) + assert.Nil(s.T(), err, "PutObject Failed") + + time.Sleep(3 * time.Second) + // Copy file + soruceURL := fmt.Sprintf("%s/%s", iu.Host, source) + dest := source + //opt := &cos.ObjectCopyOptions{} + _, _, err = s.Client.Object.Copy(context.Background(), dest, soruceURL, nil) + assert.Nil(s.T(), err, "PutObjectCopy Failed") + + // Check content + resp, err := s.Client.Object.Get(context.Background(), dest, nil) + assert.Nil(s.T(), err, "GetObject Failed") + bs, _ := ioutil.ReadAll(resp.Body) + resp.Body.Close() + result := string(bs) + assert.Equal(s.T(), expected, result, "PutObjectCopy Failed, wrong content") +} + +func (s *CosTestSuite) TestCreateAbortMultipartUpload() { + name := "test_multipart.txt" + v, _, err := s.Client.Object.InitiateMultipartUpload(context.Background(), name, nil) + assert.Nil(s.T(), err, "InitiateMultipartUpload Failed") + + _, err = s.Client.Object.AbortMultipartUpload(context.Background(), name, v.UploadID) + assert.Nil(s.T(), err, "AbortMultipartUpload Failed") +} + +func (s *CosTestSuite) TestCreateCompleteMultipartUpload() { + name := "test/test_complete_upload.go" + v, _, err := s.Client.Object.InitiateMultipartUpload(context.Background(), name, nil) + uploadID := v.UploadID + blockSize := 1024 * 1024 * 3 + + opt := &cos.CompleteMultipartUploadOptions{} + for i := 1; i < 5; i++ { + b := make([]byte, blockSize) + _, err := rand.Read(b) + content := fmt.Sprintf("%X", b) + f := strings.NewReader(content) + + resp, err := s.Client.Object.UploadPart( + context.Background(), name, uploadID, i, f, nil, + ) + assert.Nil(s.T(), err, "UploadPart Failed") + etag := resp.Header.Get("Etag") + opt.Parts = append(opt.Parts, cos.Object{ + PartNumber: i, ETag: etag}, + ) + } + + _, _, err = s.Client.Object.CompleteMultipartUpload( + context.Background(), name, uploadID, opt, + ) + assert.Nil(s.T(), err, "CompleteMultipartUpload Failed") +} + +// End of api test + +// All methods that begin with "Test" are run as tests within a +// suite. +// In order for 'go test' to run this suite, we need to create +// a normal test function and pass our suite to suite.Run +func TestCosTestSuite(t *testing.T) { + suite.Run(t, new(CosTestSuite)) +} + +func (s *CosTestSuite) TearDownSuite() { + // Clean the file in bucket + r, _, err := s.Client.Bucket.ListMultipartUploads(context.Background(), nil) + assert.Nil(s.T(), err, "ListMultipartUploads Failed") + for _, p := range r.Uploads { + // Abort + _, err = s.Client.Object.AbortMultipartUpload(context.Background(), p.Key, p.UploadID) + assert.Nil(s.T(), err, "AbortMultipartUpload Failed") + } + + // Delete objects + opt := &cos.BucketGetOptions{ + MaxKeys: 500, + } + v, _, err := s.Client.Bucket.Get(context.Background(), opt) + assert.Nil(s.T(), err, "GetBucket Failed") + for _, c := range v.Contents { + _, err := s.Client.Object.Delete(context.Background(), c.Key) + assert.Nil(s.T(), err, "DeleteObject Failed") + } + + fmt.Println("tear down~") + +} diff --git a/example/object/get.go b/example/object/get.go index 1834adb..8369f4f 100644 --- a/example/object/get.go +++ b/example/object/get.go @@ -6,6 +6,7 @@ import ( "net/url" "os" + "io" "io/ioutil" "net/http" @@ -21,6 +22,8 @@ func main() { Transport: &cos.AuthorizationTransport{ SecretID: os.Getenv("COS_SECRETID"), SecretKey: os.Getenv("COS_SECRETKEY"), + SecretID: ak, + SecretKey: sk, Transport: &debug.DebugRequestTransport{ RequestHeader: true, RequestBody: true, @@ -30,6 +33,7 @@ func main() { }, }) + // Case1 Download object into ReadCloser(). the body needs to be closed name := "test/hello.txt" resp, err := c.Object.Get(context.Background(), name, nil) if err != nil { @@ -39,7 +43,26 @@ func main() { resp.Body.Close() fmt.Printf("%s\n", string(bs)) - // range + // Case2 Download object to local file. the body needs to be closed + fd, err := os.OpenFile("hello.txt", os.O_WRONLY|os.O_CREATE, 0660) + if err != nil { + panic(err) + } + defer fd.Close() + resp, err = c.Object.Get(context.Background(), name, nil) + if err != nil { + panic(err) + } + io.Copy(fd, resp.Body) + resp.Body.Close() + + // Case3 Download object to local file path + err = c.Object.GetToFile(context.Background(), name, "hello_1.txt", nil) + if err != nil { + panic(err) + } + + // Case4 Download object with range header, can used to concurrent download opt := &cos.ObjectGetOptions{ ResponseContentType: "text/html", Range: "bytes=0-3", diff --git a/example/object/put.go b/example/object/put.go index 4ed25c4..0ed1a09 100644 --- a/example/object/put.go +++ b/example/object/put.go @@ -28,6 +28,7 @@ func main() { }, }) + // Case1 normal put object name := "test/objectPut.go" f := strings.NewReader("test") @@ -36,6 +37,7 @@ func main() { panic(err) } + // Case2 put object with the options name = "test/put_option.go" f = strings.NewReader("test xxx") opt := &cos.ObjectPutOptions{ @@ -51,4 +53,11 @@ func main() { if err != nil { panic(err) } + + // Case3 put object by local file + _, err = c.Object.PutFromFile(context.Background(), name, "./test10M", nil) + if err != nil { + panic(err) + } + } diff --git a/example/test.sh b/example/test.sh index 8dd2729..b5c8067 100644 --- a/example/test.sh +++ b/example/test.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -function run() { +run() { go run "$@" if [ $? -ne 0 ] diff --git a/object.go b/object.go index 69c700b..9fd5826 100644 --- a/object.go +++ b/object.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "net/http" + "os" "strings" ) @@ -41,6 +42,29 @@ func (s *ObjectService) Get(ctx context.Context, name string, opt *ObjectGetOpti return resp, err } +// GetToFile download the object to local file +func (s *ObjectService) GetToFile(ctx context.Context, name, localpath string, opt *ObjectGetOptions) (*Response, error) { + resp, err := s.Get(context.Background(), name, opt) + if err != nil { + return resp, err + } + defer resp.Body.Close() + + // If file exist, overwrite it + fd, err := os.OpenFile(localpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660) + if err != nil { + return resp, err + } + + _, err = io.Copy(fd, resp.Body) + fd.Close() + if err != nil { + return resp, err + } + + return resp, nil +} + // ObjectPutHeaderOptions the options of header of the put object type ObjectPutHeaderOptions struct { CacheControl string `header:"Cache-Control,omitempty" url:"-"` @@ -81,6 +105,17 @@ func (s *ObjectService) Put(ctx context.Context, name string, r io.Reader, opt * return resp, err } +// PutFromFile put object from local file +func (s *ObjectService) PutFromFile(ctx context.Context, name string, filePath string, opt *ObjectPutOptions) (*Response, error) { + fd, err := os.Open(filePath) + if err != nil { + return nil, err + } + defer fd.Close() + + return s.Put(ctx, name, fd, opt) +} + // ObjectCopyHeaderOptions is the head option of the Copy type ObjectCopyHeaderOptions struct { XCosMetadataDirective string `header:"x-cos-metadata-directive,omitempty" url:"-" xml:"-"` diff --git a/object_part.go b/object_part.go index e2201be..7e95d14 100644 --- a/object_part.go +++ b/object_part.go @@ -121,6 +121,22 @@ type CompleteMultipartUploadResult struct { ETag string } +// ObjectList can used for sort the parts which needs in complete upload part +// sort.Sort(cos.ObjectList(opt.Parts)) +type ObjectList []Object + +func (o ObjectList) Len() int { + return len(o) +} + +func (o ObjectList) Swap(i, j int) { + o[i], o[j] = o[j], o[i] +} + +func (o ObjectList) Less(i, j int) bool { // rewrite the Less method from small to big + return o[i].PartNumber < o[j].PartNumber +} + // CompleteMultipartUpload 用来实现完成整个分块上传。当您已经使用Upload Parts上传所有块以后，你可以用该API完成上传。 // 在使用该API时，您必须在Body中给出每一个块的PartNumber和ETag，用来校验块的准确性。 // diff --git a/vendor/github.com/google/go-querystring b/vendor/github.com/google/go-querystring new file mode 160000 index 0000000..44c6ddd --- /dev/null +++ b/vendor/github.com/google/go-querystring @@ -0,0 +1 @@ +Subproject commit 44c6ddd0a2342c386950e880b658017258da92fc diff --git a/vendor/github.com/mozillazg/go-httpheader b/vendor/github.com/mozillazg/go-httpheader new file mode 160000 index 0000000..61f2392 --- /dev/null +++ b/vendor/github.com/mozillazg/go-httpheader @@ -0,0 +1 @@ +Subproject commit 61f2392c3317b60616c9dcb10d0a4cfef131fe62 diff --git a/vendor/github.com/stretchr/testify b/vendor/github.com/stretchr/testify new file mode 160000 index 0000000..363ebb2 --- /dev/null +++ b/vendor/github.com/stretchr/testify @@ -0,0 +1 @@ +Subproject commit 363ebb24d041ccea8068222281c2e963e997b9dc From 77bfe2d0c7a77ad2807f8fe43d70bc95a45508bc Mon Sep 17 00:00:00 2001 From: toranger Date: Thu, 31 Jan 2019 10:47:05 +0800 Subject: [PATCH 2/8] fix travis yml and test --- .travis.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 39bf574..f96f7e7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,5 @@ language: go go: -- '1.6' - '1.7' - '1.8' - '1.9' @@ -9,17 +8,17 @@ go: sudo: false before_install: - go get github.com/mattn/goveralls + go get github.com/stretchr/testify install: - go get - go build script: - make test - make ci-test -- go test -coverprofile=cover.out github.com/tencentyun/cos-go-sdk-v5 -- "$HOME/gopath/bin/goveralls -service=travis-ci -coverprofile=cover.out" +- go test -coverprofile=cover.out github.com/toranger/cos-go-sdk-v5 +- "${TRAVIS_HOME}/gopath/bin/goveralls -service=travis-ci -coverprofile=cover.out" matrix: allow_failures: - - go: 1.6 - go: 1.7 - go: master env: From e4701195aaa0f5963442a498b16141e4dcc8d21a Mon Sep 17 00:00:00 2001 From: toranger Date: Thu, 31 Jan 2019 10:50:53 +0800 Subject: [PATCH 3/8] fix travis yml and test --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f96f7e7..e7228e3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,7 @@ go: sudo: false before_install: - go get github.com/mattn/goveralls - go get github.com/stretchr/testify +- go get github.com/stretchr/testify install: - go get - go build From 72f6fe32b96d0e29bb9763664de059c8de46045f Mon Sep 17 00:00:00 2001 From: toranger Date: Thu, 31 Jan 2019 11:01:45 +0800 Subject: [PATCH 4/8] Fix yml and import package --- .travis.yml | 1 + costesting/ci_test.go | 9 +-------- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index e7228e3..5202cd9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,6 +12,7 @@ before_install: install: - go get - go build +- go build github.com/mattn/goveralls script: - make test - make ci-test diff --git a/costesting/ci_test.go b/costesting/ci_test.go index e2ecdda..7176067 100644 --- a/costesting/ci_test.go +++ b/costesting/ci_test.go @@ -6,8 +6,7 @@ import ( "fmt" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" - "github.com/tencentyun/cos-go-sdk-v5" - "github.com/tencentyun/cos-go-sdk-v5/debug" + "github.com/toranger/cos-go-sdk-v5" "io/ioutil" "math/rand" "net/http" @@ -70,12 +69,6 @@ func (s *CosTestSuite) SetupSuite() { Transport: &cos.AuthorizationTransport{ SecretID: os.Getenv("COS_SECRETID"), SecretKey: os.Getenv("COS_SECRETKEY"), - Transport: &debug.DebugRequestTransport{ - // RequestHeader: true, - // RequestBody: true, - // ResponseHeader: true, - // ResponseBody: true, - }, }, }) From 9ebd6456b6aa788d0d2e97f4a8fb652c1b9a1976 Mon Sep 17 00:00:00 2001 From: toranger Date: Thu, 31 Jan 2019 12:00:43 +0800 Subject: [PATCH 5/8] fix --- costesting/ci_test.go | 42 ++++++++++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/costesting/ci_test.go b/costesting/ci_test.go index 7176067..7fb9536 100644 --- a/costesting/ci_test.go +++ b/costesting/ci_test.go @@ -93,7 +93,7 @@ func (s *CosTestSuite) TestGetService() { // Bucket API func (s *CosTestSuite) TestPutHeadDeleteBucket() { - u := "http://gosdkbuckettest-" + s.Appid + ".cos.ap-beijing-1.myqcloud.com" + u := "http://gosdkbuckettest-" + time.Now().Format(time.RFC3339) + "-" + s.Appid + ".cos.ap-beijing-1.myqcloud.com" iu, _ := url.Parse(u) ib := &cos.BaseURL{BucketURL: iu} client := cos.NewClient(ib, &http.Client{ @@ -110,6 +110,7 @@ func (s *CosTestSuite) TestPutHeadDeleteBucket() { } time.Sleep(3 * time.Second) + _, err = client.Bucket.Head(context.Background()) assert.Nil(s.T(), err, "HeadBucket Failed") @@ -203,7 +204,8 @@ func (s *CosTestSuite) TestPutGetDeleteLifeCycle() { func (s *CosTestSuite) TestListMultipartUploads() { // Create new upload - name := "test_multipart.txt" + name := "test_multipart" + time.Now().Format(time.RFC3339) + flag := false v, _, err := s.Client.Object.InitiateMultipartUpload(context.Background(), name, nil) assert.Nil(s.T(), err, "InitiateMultipartUpload Failed") id := v.UploadID @@ -211,11 +213,13 @@ func (s *CosTestSuite) TestListMultipartUploads() { // List r, _, err := s.Client.Bucket.ListMultipartUploads(context.Background(), nil) assert.Nil(s.T(), err, "ListMultipartUploads Failed") - assert.Equal(s.T(), 1, len(r.Uploads), "ListMultipartUploads wrong number uploads") for _, p := range r.Uploads { - assert.Equal(s.T(), name, p.Key, "ListMultipartUploads wrong key") - assert.Equal(s.T(), id, p.UploadID, "ListMultipartUploads wrong uploadid") + if p.Key == name { + assert.Equal(s.T(), id, p.UploadID, "ListMultipartUploads wrong uploadid") + flag = true + } } + assert.Equal(s.T(), true, flag, "ListMultipartUploads wrong key") // Abort _, err = s.Client.Object.AbortMultipartUpload(context.Background(), name, id) @@ -224,7 +228,7 @@ func (s *CosTestSuite) TestListMultipartUploads() { // Object API func (s *CosTestSuite) TestPutHeadGetDeleteObject_10MB() { - name := "test/objectPut.go" + name := "test/objectPut" + time.Now().Format(time.RFC3339) b := make([]byte, 1024*1024*10) _, err := rand.Read(b) content := fmt.Sprintf("%X", b) @@ -242,12 +246,12 @@ func (s *CosTestSuite) TestPutHeadGetDeleteObject_10MB() { func (s *CosTestSuite) TestPutGetDeleteObjectByFile_10MB() { // Create tmp file - filePath := "tmpfile" + filePath := "tmpfile" + time.Now().Format(time.RFC3339) newfile, err := os.Create(filePath) assert.Nil(s.T(), err, "create tmp file Failed") defer newfile.Close() - name := "test/objectPutByFile.go" + name := "test/objectPutByFile" + time.Now().Format(time.RFC3339) b := make([]byte, 1024*1024*10) _, err = rand.Read(b) @@ -269,16 +273,17 @@ func (s *CosTestSuite) TestPutGetDeleteObjectByFile_10MB() { func (s *CosTestSuite) TestPutGetDeleteObjectSpecialName() { f := strings.NewReader("test") - _, err := s.Client.Object.Put(context.Background(), s.SepFileName, f, nil) + name := s.SepFileName + time.Now().Format(time.RFC3339) + _, err := s.Client.Object.Put(context.Background(), name, f, nil) assert.Nil(s.T(), err, "PutObject Failed") - resp, err := s.Client.Object.Get(context.Background(), s.SepFileName, nil) + resp, err := s.Client.Object.Get(context.Background(), name, nil) assert.Nil(s.T(), err, "GetObject Failed") defer resp.Body.Close() bs, _ := ioutil.ReadAll(resp.Body) assert.Equal(s.T(), "test", string(bs), "GetObject failed content wrong") - _, err = s.Client.Object.Delete(context.Background(), s.SepFileName) + _, err = s.Client.Object.Delete(context.Background(), name) assert.Nil(s.T(), err, "DeleteObject Failed") } @@ -300,7 +305,7 @@ func (s *CosTestSuite) TestPutObjectToNonExistBucket() { } func (s *CosTestSuite) TestPutGetObjectACL() { - name := "test/objectACL.go" + name := "test/objectACL.go" + time.Now().Format(time.RFC3339) f := strings.NewReader("test") _, err := s.Client.Object.Put(context.Background(), name, f, nil) assert.Nil(s.T(), err, "PutObject Failed") @@ -344,7 +349,7 @@ func (s *CosTestSuite) TestCopyObject() { assert.Nil(s.T(), err, "PutBucket Failed") } - source := "test/objectMove1.go" + source := "test/objectMove1" + time.Now().Format(time.RFC3339) expected := "test" f := strings.NewReader(expected) @@ -369,7 +374,7 @@ func (s *CosTestSuite) TestCopyObject() { } func (s *CosTestSuite) TestCreateAbortMultipartUpload() { - name := "test_multipart.txt" + name := "test_multipart" + time.Now().Format(time.RFC3339) v, _, err := s.Client.Object.InitiateMultipartUpload(context.Background(), name, nil) assert.Nil(s.T(), err, "InitiateMultipartUpload Failed") @@ -378,13 +383,13 @@ func (s *CosTestSuite) TestCreateAbortMultipartUpload() { } func (s *CosTestSuite) TestCreateCompleteMultipartUpload() { - name := "test/test_complete_upload.go" + name := "test/test_complete_upload" + time.Now().Format(time.RFC3339) v, _, err := s.Client.Object.InitiateMultipartUpload(context.Background(), name, nil) uploadID := v.UploadID blockSize := 1024 * 1024 * 3 opt := &cos.CompleteMultipartUploadOptions{} - for i := 1; i < 5; i++ { + for i := 1; i < 3; i++ { b := make([]byte, blockSize) _, err := rand.Read(b) content := fmt.Sprintf("%X", b) @@ -403,6 +408,11 @@ func (s *CosTestSuite) TestCreateCompleteMultipartUpload() { _, _, err = s.Client.Object.CompleteMultipartUpload( context.Background(), name, uploadID, opt, ) + + if err != nil { + _, err = s.Client.Object.AbortMultipartUpload(context.Background(), name, uploadID) + assert.Nil(s.T(), err, "AbortMultipartUpload Failed") + } assert.Nil(s.T(), err, "CompleteMultipartUpload Failed") } From c17efed0aedee60e819d3e9504720ff47958da14 Mon Sep 17 00:00:00 2001 From: toranger Date: Thu, 31 Jan 2019 14:23:34 +0800 Subject: [PATCH 6/8] fix ci_test --- costesting/ci_test.go | 50 ++++++++++++++++++++++++++------------------------ 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/costesting/ci_test.go b/costesting/ci_test.go index 7fb9536..3f5adaf 100644 --- a/costesting/ci_test.go +++ b/costesting/ci_test.go @@ -93,6 +93,7 @@ func (s *CosTestSuite) TestGetService() { // Bucket API func (s *CosTestSuite) TestPutHeadDeleteBucket() { + // Notic sometimes the bucket host can not analyis, may has i/o timeout problem u := "http://gosdkbuckettest-" + time.Now().Format(time.RFC3339) + "-" + s.Appid + ".cos.ap-beijing-1.myqcloud.com" iu, _ := url.Parse(u) ib := &cos.BaseURL{BucketURL: iu} @@ -196,8 +197,11 @@ func (s *CosTestSuite) TestPutGetDeleteLifeCycle() { } _, err := s.Client.Bucket.PutLifecycle(context.Background(), lc) assert.Nil(s.T(), err, "PutBucketLifecycle Failed") - _, _, err = s.Client.Bucket.GetLifecycle(context.Background()) - assert.Nil(s.T(), err, "GetBucketLifecycle Failed") + _, r, err := s.Client.Bucket.GetLifecycle(context.Background()) + // Might cleaned by other case concrrent + if err != nil && 404 != r.StatusCode { + assert.Nil(s.T(), err, "GetBucketLifecycle Failed") + } _, err = s.Client.Bucket.DeleteLifecycle(context.Background()) assert.Nil(s.T(), err, "DeleteBucketLifecycle Failed") } @@ -409,10 +413,6 @@ func (s *CosTestSuite) TestCreateCompleteMultipartUpload() { context.Background(), name, uploadID, opt, ) - if err != nil { - _, err = s.Client.Object.AbortMultipartUpload(context.Background(), name, uploadID) - assert.Nil(s.T(), err, "AbortMultipartUpload Failed") - } assert.Nil(s.T(), err, "CompleteMultipartUpload Failed") } @@ -428,24 +428,26 @@ func TestCosTestSuite(t *testing.T) { func (s *CosTestSuite) TearDownSuite() { // Clean the file in bucket - r, _, err := s.Client.Bucket.ListMultipartUploads(context.Background(), nil) - assert.Nil(s.T(), err, "ListMultipartUploads Failed") - for _, p := range r.Uploads { - // Abort - _, err = s.Client.Object.AbortMultipartUpload(context.Background(), p.Key, p.UploadID) - assert.Nil(s.T(), err, "AbortMultipartUpload Failed") - } - - // Delete objects - opt := &cos.BucketGetOptions{ - MaxKeys: 500, - } - v, _, err := s.Client.Bucket.Get(context.Background(), opt) - assert.Nil(s.T(), err, "GetBucket Failed") - for _, c := range v.Contents { - _, err := s.Client.Object.Delete(context.Background(), c.Key) - assert.Nil(s.T(), err, "DeleteObject Failed") - } + // r, _, err := s.Client.Bucket.ListMultipartUploads(context.Background(), nil) + // assert.Nil(s.T(), err, "ListMultipartUploads Failed") + // for _, p := range r.Uploads { + // // Abort + // _, err = s.Client.Object.AbortMultipartUpload(context.Background(), p.Key, p.UploadID) + // assert.Nil(s.T(), err, "AbortMultipartUpload Failed") + // } + + // // Delete objects + // opt := &cos.BucketGetOptions{ + // MaxKeys: 500, + // } + // v, _, err := s.Client.Bucket.Get(context.Background(), opt) + // assert.Nil(s.T(), err, "GetBucket Failed") + // for _, c := range v.Contents { + // _, err := s.Client.Object.Delete(context.Background(), c.Key) + // assert.Nil(s.T(), err, "DeleteObject Failed") + // } + + // When clean up these infos, can not solve the concurrent test problem fmt.Println("tear down~") From a6d215db23bbf110937165e79d4f50c4a91805d5 Mon Sep 17 00:00:00 2001 From: toranger Date: Thu, 31 Jan 2019 14:37:34 +0800 Subject: [PATCH 7/8] go on test --- cos.go | 2 +- costesting/ci_test.go | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/cos.go b/cos.go index bb82983..adc381f 100644 --- a/cos.go +++ b/cos.go @@ -24,7 +24,7 @@ const ( Version = "0.7.3" userAgent = "cos-go-sdk-v5/" + Version contentTypeXML = "application/xml" - defaultServiceBaseURL = "https://service.cos.myqcloud.com" + defaultServiceBaseURL = "http://service.cos.myqcloud.com" ) var bucketURLTemplate = template.Must( diff --git a/costesting/ci_test.go b/costesting/ci_test.go index 3f5adaf..6e90da9 100644 --- a/costesting/ci_test.go +++ b/costesting/ci_test.go @@ -110,6 +110,10 @@ func (s *CosTestSuite) TestPutHeadDeleteBucket() { assert.Nil(s.T(), err, "PutBucket Failed") } + if err != nil { + panic(err) + } + time.Sleep(3 * time.Second) _, err = client.Bucket.Head(context.Background()) From 0e8a940b25d27600ba63d909066e8d1a19b7e06b Mon Sep 17 00:00:00 2001 From: toranger Date: Thu, 31 Jan 2019 14:55:16 +0800 Subject: [PATCH 8/8] ci --- costesting/ci_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/costesting/ci_test.go b/costesting/ci_test.go index 6e90da9..a85fe29 100644 --- a/costesting/ci_test.go +++ b/costesting/ci_test.go @@ -94,7 +94,7 @@ func (s *CosTestSuite) TestGetService() { // Bucket API func (s *CosTestSuite) TestPutHeadDeleteBucket() { // Notic sometimes the bucket host can not analyis, may has i/o timeout problem - u := "http://gosdkbuckettest-" + time.Now().Format(time.RFC3339) + "-" + s.Appid + ".cos.ap-beijing-1.myqcloud.com" + u := "http://gosdkbuckettest-" + s.Appid + ".cos.ap-beijing-1.myqcloud.com" iu, _ := url.Parse(u) ib := &cos.BaseURL{BucketURL: iu} client := cos.NewClient(ib, &http.Client{ @@ -118,10 +118,10 @@ func (s *CosTestSuite) TestPutHeadDeleteBucket() { _, err = client.Bucket.Head(context.Background()) assert.Nil(s.T(), err, "HeadBucket Failed") - - _, err = client.Bucket.Delete(context.Background()) - assert.Nil(s.T(), err, "DeleteBucket Failed") - + if err == nil { + _, err = client.Bucket.Delete(context.Background()) + assert.Nil(s.T(), err, "DeleteBucket Failed") + } } func (s *CosTestSuite) TestPutBucketACLIllegal() {