diff --git a/batch.go b/batch.go new file mode 100644 index 0000000..390d20c --- /dev/null +++ b/batch.go @@ -0,0 +1,262 @@ +package cos + +import ( + "context" + "encoding/xml" + "fmt" + "net/http" +) + +type BatchService service + +type BatchRequestHeaders struct { + XCosAppid int `header:"x-cos-appid" xml:"-" url:"-"` + ContentLength string `header:"Content-Length,omitempty" xml:"-" url:"-"` + ContentType string `header:"Content-Type,omitempty" xml:"-" url:"-"` + Headers *http.Header `header:"-" xml:"-", url:"-"` +} + +// BatchProgressSummary +type BatchProgressSummary struct { + NumberOfTasksFailed int `xml:"NumberOfTasksFailed" header:"-" url:"-"` + NumberOfTasksSucceeded int `xml:"NumberOfTasksSucceeded" header:"-" url:"-"` + TotalNumberOfTasks int `xml:"TotalNumberOfTasks" header:"-" url:"-"` +} + +// BatchJobReport +type BatchJobReport struct { + Bucket string `xml:"Bucket" header:"-" url:"-"` + Enabled string `xml:"Enabled" header:"-" url:"-"` + Format string `xml:"Format" header:"-" url:"-"` + Prefix string `xml:"Prefix,omitempty" header:"-" url:"-"` + ReportScope string `xml:"ReportScope" header:"-" url:"-"` +} + +// BatchJobOperationCopy +type BatchMetadata struct { + Key string `xml:"Key" header:"-" url:"-"` + Value string `xml:"Value" header:"-" url:"-"` +} +type BatchNewObjectMetadata struct { + CacheControl string `xml:"CacheControl,omitempty" header:"-" url:"-"` + ContentDisposition string `xml:"ContentDisposition,omitempty" header:"-" url:"-"` + ContentEncoding string `xml:"ContentEncoding,omitempty" header:"-" url:"-"` + ContentType string `xml:"ContentType,omitempty" header:"-" url:"-"` + HttpExpiresDate string `xml:"HttpExpiresDate,omitempty" header:"-" url:"-"` + SSEAlgorithm string `xml:"SSEAlgorithm,omitempty" header:"-" url:"-"` + UserMetadata []BatchMetadata `xml:"UserMetadata>member,omitempty" header:"-" url:"-"` +} +type BatchGrantee struct { + DisplayName string `xml:"DisplayName,omitempty" header:"-" url:"-"` + Identifier string `xml:"Identifier" header:"-" url:"-"` + TypeIdentifier string `xml:"TypeIdentifier" header:"-" url:"-"` +} +type BatchCOSGrant struct { + Grantee *BatchGrantee `xml:"Grantee" header:"-" url:"-"` + Permission string `xml:"Permission" header:"-" url:"-"` +} +type BatchAccessControlGrants struct { + COSGrants *BatchCOSGrant `xml:"COSGrant,omitempty" header:"-" url:"-"` +} +type BatchJobOperationCopy struct { + AccessControlGrants *BatchAccessControlGrants `xml:"AccessControlGrants,omitempty" header:"-" url:"-"` + CannedAccessControlList string `xml:"CannedAccessControlList,omitempty" header:"-" url:"-"` + MetadataDirective string `xml:"MetadataDirective,omitempty" header:"-" url:"-"` + ModifiedSinceConstraint int64 `xml:"ModifiedSinceConstraint,omitempty" header:"-" url:"-"` + UnModifiedSinceConstraint int64 `xml:"UnModifiedSinceConstraint,omitempty" header:"-" url:"-"` + NewObjectMetadata *BatchNewObjectMetadata `xml:"NewObjectMetadata,omitempty" header:"-" url:"-"` + StorageClass string `xml:"StorageClass,omitempty" header:"-" url:"-"` + TargetResource string `xml:"TargetResource" header:"-" url:"-"` +} + +// BatchJobOperation +type BatchJobOperation struct { + PutObjectCopy *BatchJobOperationCopy `xml:"COSPutObjectCopy,omitempty" header:"-" url:"-"` +} + +// BatchJobManifest +type BatchJobManifestLocation struct { + ETag string `xml:"ETag" header:"-" url:"-"` + ObjectArn string `xml:"ObjectArn" header:"-" url:"-"` + ObjectVersionId string `xml:"ObjectVersionId,omitempty" header:"-" url:"-"` +} +type BatchJobManifestSpec struct { + Fields []string `xml:"Fields>member,omitempty" header:"-" url:"-"` + Format string `xml:"Format" header:"-" url:"-"` +} +type BatchJobManifest struct { + Location *BatchJobManifestLocation `xml:"Location" header:"-" url:"-"` + Spec *BatchJobManifestSpec `xml:"Spec" header:"-" url:"-"` +} + +type BatchCreateJobOptions struct { + XMLName xml.Name `xml:"CreateJobRequest" header:"-" url:"-"` + ClientRequestToken string `xml:"ClientRequestToken" header:"-" url:"-"` + ConfirmationRequired string `xml:"ConfirmationRequired,omitempty" header:"-" url:"-"` + Description string `xml:"Description,omitempty" header:"-" url:"-"` + Manifest *BatchJobManifest `xml:"Manifest" header:"-" url:"-"` + Operation *BatchJobOperation `xml:"Operation" header:"-" url:"-"` + Priority int `xml:"Priority" header:"-" url:"-"` + Report *BatchJobReport `xml:"Report" header:"-" url:"-"` + RoleArn string `xml:"RoleArn" header:"-" url:"-"` +} + +type BatchCreateJobResult struct { + XMLName xml.Name `xml:"CreateJobResult"` + JobId string `xml:"JobId,omitempty"` +} + +func processETag(opt *BatchCreateJobOptions) *BatchCreateJobOptions { + if opt != nil && opt.Manifest != nil && opt.Manifest.Location != nil { + opt.Manifest.Location.ETag = "" + opt.Manifest.Location.ETag + "" + } + return opt +} + +func (s *BatchService) CreateJob(ctx context.Context, opt *BatchCreateJobOptions, headers *BatchRequestHeaders) (*BatchCreateJobResult, *Response, error) { + var res BatchCreateJobResult + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.BatchURL, + uri: "/jobs", + method: http.MethodPost, + optHeader: headers, + body: opt, + result: &res, + } + + resp, err := s.client.send(ctx, &sendOpt) + return &res, resp, err +} + +type BatchJobFailureReasons struct { + FailureCode string `xml:"FailureCode" header:"-" url:"-"` + FailureReason string `xml:"FailureReason" header:"-" url:"-"` +} + +type BatchDescribeJob struct { + ConfirmationRequired string `xml:"ConfirmationRequired,omitempty" header:"-" url:"-"` + CreationTime string `xml:"CreationTime,omitempty" header:"-" url:"-"` + Description string `xml:"Description,omitempty" header:"-" url:"-"` + FailureReasons *BatchJobFailureReasons `xml:"FailureReasons>JobFailure,omitempty" header:"-" url:"-"` + JobId string `xml:"JobId" header:"-" url:"-"` + Manifest *BatchJobManifest `xml:"Manifest" header:"-" url:"-"` + Operation *BatchJobOperation `xml:"Operation" header:"-" url:"-"` + Priority int `xml:"Priority" header:"-" url:"-"` + ProgressSummary *BatchProgressSummary `xml:"ProgressSummary" header:"-" url:"-"` + Report *BatchJobReport `xml:"Report,omitempty" header:"-" url:"-"` + RoleArn string `xml:"RoleArn,omitempty" header:"-" url:"-"` + Status string `xml:"Status,omitempty" header:"-" url:"-"` + StatusUpdateReason string `xml:"StatusUpdateReason,omitempty" header:"-" url:"-"` + SuspendedCause string `xml:"SuspendedCause,omitempty" header:"-" url:"-"` + SuspendedDate string `xml:"SuspendedDate,omitempty" header:"-" url:"-"` + TerminationDate string `xml:"TerminationDate,omitempty" header:"-" url:"-"` +} +type BatchDescribeJobResult struct { + XMLName xml.Name `xml:"DescribeJobResult"` + Job *BatchDescribeJob `xml:"Job,omitempty"` +} + +func (s *BatchService) DescribeJob(ctx context.Context, id string, headers *BatchRequestHeaders) (*BatchDescribeJobResult, *Response, error) { + var res BatchDescribeJobResult + u := fmt.Sprintf("/jobs/%s", id) + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.BatchURL, + uri: u, + method: http.MethodGet, + optHeader: headers, + result: &res, + } + resp, err := s.client.send(ctx, &sendOpt) + return &res, resp, err +} + +type BatchListJobsOptions struct { + JobStatuses string `url:"jobStatuses,omitempty" header:"-" xml:"-"` + MaxResults int `url:"maxResults,omitempty" header:"-" xml:"-"` + NextToken string `url:"nextToken,omitempty" header:"-" xml:"-"` +} + +type BatchListJobsMember struct { + CreationTime string `xml:"CreationTime,omitempty" header:"-" url:"-"` + Description string `xml:"Description,omitempty" header:"-" url:"-"` + JobId string `xml:"JobId,omitempty" header:"-" url:"-"` + Operation string `xml:"Operation,omitempty" header:"-" url:"-"` + Priority int `xml:"Priority,omitempty" header:"-" url:"-"` + ProgressSummary *BatchProgressSummary `xml:"ProgressSummary,omitempty" header:"-" url:"-"` + Status string `xml:"Status,omitempty" header:"-" url:"-"` + TerminationDate string `xml:"TerminationDate,omitempty" header:"-" url:"-"` +} +type BatchListJobs struct { + Members []BatchListJobsMember `xml:"member,omitempty" header:"-" url:"-"` +} +type BatchListJobsResult struct { + XMLName xml.Name `xml:"ListJobsResult"` + Jobs *BatchListJobs `xml:"Jobs,omitempty"` + NextToken string `xml:"NextToken,omitempty"` +} + +func (s *BatchService) ListJobs(ctx context.Context, opt *BatchListJobsOptions, headers *BatchRequestHeaders) (*BatchListJobsResult, *Response, error) { + var res BatchListJobsResult + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.BatchURL, + uri: "/jobs", + method: http.MethodGet, + optQuery: opt, + optHeader: headers, + result: &res, + } + resp, err := s.client.send(ctx, &sendOpt) + return &res, resp, err +} + +type BatchUpdatePriorityOptions struct { + JobId string `url:"-" header:"-" xml:"-"` + Priority int `url:"priority" header:"-" xml:"-"` +} +type BatchUpdatePriorityResult struct { + XMLName xml.Name `xml:"UpdateJobPriorityResult"` + JobId string `xml:"JobId,omitempty"` + Priority int `xml:"Priority,omitempty"` +} + +func (s *BatchService) UpdateJobPriority(ctx context.Context, opt *BatchUpdatePriorityOptions, headers *BatchRequestHeaders) (*BatchUpdatePriorityResult, *Response, error) { + u := fmt.Sprintf("/jobs/%s/priority", opt.JobId) + var res BatchUpdatePriorityResult + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.BatchURL, + uri: u, + method: http.MethodPost, + optQuery: opt, + optHeader: headers, + result: &res, + } + resp, err := s.client.send(ctx, &sendOpt) + return &res, resp, err +} + +type BatchUpdateStatusOptions struct { + JobId string `header:"-" url:"-" xml:"-"` + RequestedJobStatus string `url:"requestedJobStatus" header:"-" xml:"-"` + StatusUpdateReason string `url:"statusUpdateReason,omitempty" header:"-", xml:"-"` +} +type BatchUpdateStatusResult struct { + XMLName xml.Name `xml:"UpdateJobStatusResult"` + JobId string `xml:"JobId,omitempty"` + Status string `xml:"Status,omitempty"` + StatusUpdateReason string `xml:"StatusUpdateReason,omitempty"` +} + +func (s *BatchService) UpdateJobStatus(ctx context.Context, opt *BatchUpdateStatusOptions, headers *BatchRequestHeaders) (*BatchUpdateStatusResult, *Response, error) { + u := fmt.Sprintf("/jobs/%s/status", opt.JobId) + var res BatchUpdateStatusResult + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.BatchURL, + uri: u, + method: http.MethodPost, + optQuery: opt, + optHeader: headers, + result: &res, + } + resp, err := s.client.send(ctx, &sendOpt) + return &res, resp, err +} diff --git a/batch_test.go b/batch_test.go new file mode 100644 index 0000000..8421486 --- /dev/null +++ b/batch_test.go @@ -0,0 +1,389 @@ +package cos + +import ( + "context" + "encoding/xml" + "fmt" + "net/http" + "reflect" + "testing" + + "github.com/google/uuid" +) + +func TestBatchService_CreateJob(t *testing.T) { + setup() + defer teardown() + uuid_str := uuid.New().String() + opt := &BatchCreateJobOptions{ + ClientRequestToken: uuid_str, + Description: "test batch", + Manifest: &BatchJobManifest{ + Location: &BatchJobManifestLocation{ + ETag: "15150651828fa9cdcb8356b6d1c7638b", + ObjectArn: "qcs::cos:ap-chengdu:uid/1250000000:sourcebucket-1250000000/manifests/batch-copy-manifest.csv", + }, + Spec: &BatchJobManifestSpec{ + Fields: []string{"Bucket", "Key"}, + Format: "COSBatchOperations_CSV_V1", + }, + }, + Operation: &BatchJobOperation{ + PutObjectCopy: &BatchJobOperationCopy{ + TargetResource: "qcs::cos:ap-chengdu:uid/1250000000:destinationbucket-1250000000", + }, + }, + Priority: 1, + Report: &BatchJobReport{ + Bucket: "qcs::cos:ap-chengdu:uid/1250000000:sourcebucket-1250000000", + Enabled: "true", + Format: "Report_CSV_V1", + Prefix: "job-result", + ReportScope: "AllTasks", + }, + RoleArn: "qcs::cam::uin/100000000001:roleName/COS_Batch_QcsRole", + } + + mux.HandleFunc("/jobs", func(w http.ResponseWriter, r *http.Request) { + testHeader(t, r, "x-cos-appid", "1250000000") + testMethod(t, r, http.MethodPost) + v := new(BatchCreateJobOptions) + xml.NewDecoder(r.Body).Decode(v) + + want := opt + want.XMLName = xml.Name{Local: "CreateJobRequest"} + if !reflect.DeepEqual(v, want) { + t.Errorf("Batch.CreateJob request body: %+v, want %+v", v, want) + } + fmt.Fprint(w, ` + + 53dc6228-c50b-46f7-8ad7-65e7159f1aae +`) + }) + + headers := &BatchRequestHeaders{ + XCosAppid: 1250000000, + } + ref, _, err := client.Batch.CreateJob(context.Background(), opt, headers) + if err != nil { + t.Fatalf("Batch.CreateJob returned error: %v", err) + } + + want := &BatchCreateJobResult{ + XMLName: xml.Name{Local: "CreateJobResult"}, + JobId: "53dc6228-c50b-46f7-8ad7-65e7159f1aae", + } + + if !reflect.DeepEqual(ref, want) { + t.Errorf("Batch.CreateJob returned %+v, want %+v", ref, want) + } + +} + +func TestBatchService_DescribeJob(t *testing.T) { + setup() + defer teardown() + + mux.HandleFunc("/jobs/53dc6228-c50b-46f7-8ad7-65e7159f1aae", func(w http.ResponseWriter, r *http.Request) { + testHeader(t, r, "x-cos-appid", "1250000000") + testMethod(t, r, http.MethodGet) + + fmt.Fprint(w, ` + + + false + 2019-12-19T18:00:30Z + example-job + + + + + + + 53dc6228-c50b-46f7-8ad7-65e7159f1aae + + + "15150651828fa9cdcb8356b6d1c7638b" + qcs::cos:ap-chengdu:uid/1250000000:sourcebucket-1250000000/manifests/batch-copy-manifest.csv + + + + Bucket + Key + + COSBatchOperations_CSV_V1 + + + + + qcs::cos:ap-chengdu:uid/1250000000:destinationbucket-1250000000 + + + 10 + + 0 + 10 + 10 + + + qcs::cos:ap-chengdu:uid/1250000000:sourcebucket-1250000000 + true + Report_CSV_V1 + job-result + AllTasks + + qcs::cam::uin/100000000001:roleName/COS_Batch_QcsRole + Complete + Job complete + 2019-12-19T18:00:42Z + +`) + }) + + headers := &BatchRequestHeaders{ + XCosAppid: 1250000000, + } + ref, _, err := client.Batch.DescribeJob(context.Background(), "53dc6228-c50b-46f7-8ad7-65e7159f1aae", headers) + if err != nil { + t.Fatalf("Batch.DescribeJob returned error: %v", err) + } + + want := &BatchDescribeJobResult{ + XMLName: xml.Name{Local: "DescribeJobResult"}, + Job: &BatchDescribeJob{ + ConfirmationRequired: "false", + CreationTime: "2019-12-19T18:00:30Z", + Description: "example-job", + FailureReasons: &BatchJobFailureReasons{}, + JobId: "53dc6228-c50b-46f7-8ad7-65e7159f1aae", + Manifest: &BatchJobManifest{ + Location: &BatchJobManifestLocation{ + ETag: "\"15150651828fa9cdcb8356b6d1c7638b\"", + ObjectArn: "qcs::cos:ap-chengdu:uid/1250000000:sourcebucket-1250000000/manifests/batch-copy-manifest.csv", + }, + Spec: &BatchJobManifestSpec{ + Fields: []string{"Bucket", "Key"}, + Format: "COSBatchOperations_CSV_V1", + }, + }, + Operation: &BatchJobOperation{ + PutObjectCopy: &BatchJobOperationCopy{ + TargetResource: "qcs::cos:ap-chengdu:uid/1250000000:destinationbucket-1250000000", + }, + }, + Priority: 10, + ProgressSummary: &BatchProgressSummary{ + NumberOfTasksFailed: 0, + NumberOfTasksSucceeded: 10, + TotalNumberOfTasks: 10, + }, + Report: &BatchJobReport{ + Bucket: "qcs::cos:ap-chengdu:uid/1250000000:sourcebucket-1250000000", + Enabled: "true", + Format: "Report_CSV_V1", + Prefix: "job-result", + ReportScope: "AllTasks", + }, + RoleArn: "qcs::cam::uin/100000000001:roleName/COS_Batch_QcsRole", + Status: "Complete", + StatusUpdateReason: "Job complete", + TerminationDate: "2019-12-19T18:00:42Z", + }, + } + + if !reflect.DeepEqual(ref, want) { + t.Errorf("Batch.DescribeJob returned %+v, want %+v", ref, want) + } + +} + +func TestBatchService_ListJobs(t *testing.T) { + setup() + defer teardown() + + mux.HandleFunc("/jobs", func(w http.ResponseWriter, r *http.Request) { + testHeader(t, r, "x-cos-appid", "1250000000") + testMethod(t, r, http.MethodGet) + vs := values{ + "maxResults": "2", + } + testFormValues(t, r, vs) + + fmt.Fprint(w, ` + + + + 2019-12-19T11:05:40Z + example-job + 021140d8-67ca-4e89-8089-0de9a1e40943 + COSPutObjectCopy + 10 + + 0 + 10 + 10 + + Complete + 2019-12-19T11:05:56Z + + + 2019-12-19T11:07:05Z + example-job + 066d919e-49b9-429e-b844-e17ea7b16421 + COSPutObjectCopy + 10 + + 0 + 10 + 10 + + Complete + 2019-12-19T11:07:21Z + + + 066d919e-49b9-429e-b844-e17ea7b16421 +`) + }) + + opt := &BatchListJobsOptions{ + MaxResults: 2, + } + headers := &BatchRequestHeaders{ + XCosAppid: 1250000000, + } + + ref, _, err := client.Batch.ListJobs(context.Background(), opt, headers) + if err != nil { + t.Fatalf("Batch.DescribeJob returned error: %v", err) + } + + want := &BatchListJobsResult{ + XMLName: xml.Name{Local: "ListJobsResult"}, + Jobs: &BatchListJobs{ + Members: []BatchListJobsMember{ + { + CreationTime: "2019-12-19T11:05:40Z", + Description: "example-job", + JobId: "021140d8-67ca-4e89-8089-0de9a1e40943", + Operation: "COSPutObjectCopy", + Priority: 10, + ProgressSummary: &BatchProgressSummary{ + NumberOfTasksFailed: 0, + NumberOfTasksSucceeded: 10, + TotalNumberOfTasks: 10, + }, + Status: "Complete", + TerminationDate: "2019-12-19T11:05:56Z", + }, + { + CreationTime: "2019-12-19T11:07:05Z", + Description: "example-job", + JobId: "066d919e-49b9-429e-b844-e17ea7b16421", + Operation: "COSPutObjectCopy", + Priority: 10, + ProgressSummary: &BatchProgressSummary{ + NumberOfTasksFailed: 0, + NumberOfTasksSucceeded: 10, + TotalNumberOfTasks: 10, + }, + Status: "Complete", + TerminationDate: "2019-12-19T11:07:21Z", + }, + }, + }, + NextToken: "066d919e-49b9-429e-b844-e17ea7b16421", + } + if !reflect.DeepEqual(ref, want) { + t.Errorf("Batch.ListJobs returned %+v, want %+v", ref, want) + } +} + +func TestBatchService_UpdateJobsPriority(t *testing.T) { + setup() + defer teardown() + + mux.HandleFunc("/jobs/021140d8-67ca-4e89-8089-0de9a1e40943/priority", func(w http.ResponseWriter, r *http.Request) { + testHeader(t, r, "x-cos-appid", "1250000000") + testMethod(t, r, http.MethodPost) + vs := values{ + "priority": "10", + } + testFormValues(t, r, vs) + + fmt.Fprint(w, ` + + 021140d8-67ca-4e89-8089-0de9a1e40943 + 10 +`) + }) + + opt := &BatchUpdatePriorityOptions{ + JobId: "021140d8-67ca-4e89-8089-0de9a1e40943", + Priority: 10, + } + + headers := &BatchRequestHeaders{ + XCosAppid: 1250000000, + } + + ref, _, err := client.Batch.UpdateJobPriority(context.Background(), opt, headers) + if err != nil { + t.Fatalf("Batch.UpdateJobPriority returned error: %v", err) + } + + want := &BatchUpdatePriorityResult{ + XMLName: xml.Name{Local: "UpdateJobPriorityResult"}, + JobId: "021140d8-67ca-4e89-8089-0de9a1e40943", + Priority: 10, + } + if !reflect.DeepEqual(ref, want) { + t.Errorf("Batch.UpdateJobsPriority returned %+v, want %+v", ref, want) + } +} + +func TestBatchService_UpdateJobsStatus(t *testing.T) { + setup() + defer teardown() + + mux.HandleFunc("/jobs/021140d8-67ca-4e89-8089-0de9a1e40943/status", func(w http.ResponseWriter, r *http.Request) { + testHeader(t, r, "x-cos-appid", "1250000000") + testMethod(t, r, http.MethodPost) + vs := values{ + "requestedJobStatus": "Ready", + "statusUpdateReason": "to do", + } + testFormValues(t, r, vs) + + fmt.Fprint(w, ` + + 021140d8-67ca-4e89-8089-0de9a1e40943 + Ready + to do +`) + }) + + opt := &BatchUpdateStatusOptions{ + JobId: "021140d8-67ca-4e89-8089-0de9a1e40943", + RequestedJobStatus: "Ready", + StatusUpdateReason: "to do", + } + + headers := &BatchRequestHeaders{ + XCosAppid: 1250000000, + } + + ref, _, err := client.Batch.UpdateJobStatus(context.Background(), opt, headers) + if err != nil { + t.Fatalf("Batch.UpdateJobStatus returned error: %v", err) + } + + want := &BatchUpdateStatusResult{ + XMLName: xml.Name{Local: "UpdateJobStatusResult"}, + JobId: "021140d8-67ca-4e89-8089-0de9a1e40943", + Status: "Ready", + StatusUpdateReason: "to do", + } + if !reflect.DeepEqual(ref, want) { + t.Errorf("Batch.UpdateJobsStatus returned %+v, want %+v", ref, want) + } +} diff --git a/cos.go b/cos.go index 8e50f24..132e128 100644 --- a/cos.go +++ b/cos.go @@ -39,6 +39,8 @@ type BaseURL struct { BucketURL *url.URL // 访问 service API 的基础 URL(不包含 path 部分): http://example.com ServiceURL *url.URL + // 访问 job API 的基础 URL (不包含 path 部分): http://example.com + BatchURL *url.URL } // NewBucketURL 生成 BaseURL 所需的 BucketURL @@ -78,6 +80,7 @@ type Client struct { Service *ServiceService Bucket *BucketService Object *ObjectService + Batch *BatchService } type service struct { @@ -94,6 +97,7 @@ func NewClient(uri *BaseURL, httpClient *http.Client) *Client { if uri != nil { baseURL.BucketURL = uri.BucketURL baseURL.ServiceURL = uri.ServiceURL + baseURL.BatchURL = uri.BatchURL } if baseURL.ServiceURL == nil { baseURL.ServiceURL, _ = url.Parse(defaultServiceBaseURL) @@ -108,6 +112,7 @@ func NewClient(uri *BaseURL, httpClient *http.Client) *Client { c.Service = (*ServiceService)(&c.common) c.Bucket = (*BucketService)(&c.common) c.Object = (*ObjectService)(&c.common) + c.Batch = (*BatchService)(&c.common) return c } diff --git a/cos_test.go b/cos_test.go index 0139190..f312779 100644 --- a/cos_test.go +++ b/cos_test.go @@ -32,7 +32,7 @@ func setup() { server = httptest.NewServer(mux) u, _ := url.Parse(server.URL) - client = NewClient(&BaseURL{u, u}, nil) + client = NewClient(&BaseURL{u, u, u}, nil) } // teardown closes the test HTTP server. diff --git a/costesting/ci_test.go b/costesting/ci_test.go index 37ba06b..01d5006 100644 --- a/costesting/ci_test.go +++ b/costesting/ci_test.go @@ -3,6 +3,7 @@ package cos // Basic imports import ( "context" + "errors" "fmt" "io/ioutil" "math/rand" @@ -13,6 +14,7 @@ import ( "testing" "time" + "github.com/google/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" "github.com/tencentyun/cos-go-sdk-v5" @@ -44,14 +46,24 @@ type CosTestSuite struct { SepFileName string } -// replace +// 请替换成您的账号及存储桶信息 const ( //uin - kUin = "100010805041" - //Replication Target Region, - kRepRegion = "ap-chengdu" - //Replication Target Bucket, Version management needs to be turned on beforehand + kUin = "100010805041" + kAppid = 1259654469 + + // 常规测试需要的存储桶 + kBucket = "cosgosdktest-1259654469" + kRegion = "ap-guangzhou" + + // 跨区域复制需要的目标存储桶,地域不能与kBucket存储桶相同。 kRepBucket = "cosgosdkreptest" + kRepRegion = "ap-chengdu" + + // Batch测试需要的源存储桶和目标存储桶,目前只在成都、重庆地域公测 + kBatchBucket = "testcd-1259654469" + kTargetBatchBucket = "cosgosdkreptest-1259654469" //复用了存储桶 + kBatchRegion = "ap-chengdu" ) func (s *CosTestSuite) SetupSuite() { @@ -62,11 +74,13 @@ func (s *CosTestSuite) SetupSuite() { // CI client for test interface // URL like this http://test-1253846586.cos.ap-guangzhou.myqcloud.com - u := "https://cosgosdktest-1259654469.cos.ap-guangzhou.myqcloud.com" + u := "https://" + kBucket + ".cos." + kRegion + ".myqcloud.com" + u2 := "https://" + kUin + ".cos-control." + kBatchRegion + ".myqcloud.com" // Get the region - iu, _ := url.Parse(u) - p := strings.Split(iu.Host, ".") + bucketurl, _ := url.Parse(u) + batchurl, _ := url.Parse(u2) + p := strings.Split(bucketurl.Host, ".") assert.Equal(s.T(), 5, len(p), "Bucket host is not right") s.Region = p[2] @@ -75,7 +89,7 @@ func (s *CosTestSuite) SetupSuite() { s.Bucket = pp[0] s.Appid = pp[1] - ib := &cos.BaseURL{BucketURL: iu} + ib := &cos.BaseURL{BucketURL: bucketurl, BatchURL: batchurl} s.Client = cos.NewClient(ib, &http.Client{ Transport: &cos.AuthorizationTransport{ SecretID: os.Getenv("COS_SECRETID"), @@ -105,7 +119,7 @@ func (s *CosTestSuite) TestGetService() { // Bucket API func (s *CosTestSuite) TestPutHeadDeleteBucket() { // Notic sometimes the bucket host can not analyis, may has i/o timeout problem - u := "http://gosdkbuckettest-" + s.Appid + ".cos.ap-beijing-1.myqcloud.com" + u := "http://" + "testgosdkbucket-create-head-del-" + s.Appid + ".cos." + kRegion + ".myqcloud.com" iu, _ := url.Parse(u) ib := &cos.BaseURL{BucketURL: iu} client := cos.NewClient(ib, &http.Client{ @@ -710,6 +724,112 @@ func (s *CosTestSuite) TestMultiUpload() { assert.Nil(s.T(), err, "remove tmp file failed") } +func (s *CosTestSuite) TestBatch() { + client := cos.NewClient(s.Client.BaseURL, &http.Client{ + Transport: &cos.AuthorizationTransport{ + SecretID: os.Getenv("COS_SECRETID"), + SecretKey: os.Getenv("COS_SECRETKEY"), + }, + }) + + source_name := "test/1.txt" + sf := strings.NewReader("batch test content") + _, err := client.Object.Put(context.Background(), source_name, sf, nil) + assert.Nil(s.T(), err, "object put Failed") + + manifest_name := "test/manifest.csv" + f := strings.NewReader(kBatchBucket + "," + source_name) + resp, err := client.Object.Put(context.Background(), manifest_name, f, nil) + assert.Nil(s.T(), err, "object put Failed") + etag := resp.Header.Get("ETag") + + uuid_str := uuid.New().String() + opt := &cos.BatchCreateJobOptions{ + ClientRequestToken: uuid_str, + ConfirmationRequired: "true", + Description: "test batch", + Manifest: &cos.BatchJobManifest{ + Location: &cos.BatchJobManifestLocation{ + ETag: etag, + ObjectArn: "qcs::cos:" + kBatchRegion + ":uid/" + s.Appid + ":" + kBatchBucket + "/" + manifest_name, + }, + Spec: &cos.BatchJobManifestSpec{ + Fields: []string{"Bucket", "Key"}, + Format: "COSBatchOperations_CSV_V1", + }, + }, + Operation: &cos.BatchJobOperation{ + PutObjectCopy: &cos.BatchJobOperationCopy{ + TargetResource: "qcs::cos:" + kBatchRegion + ":uid/" + s.Appid + ":" + kTargetBatchBucket, + }, + }, + Priority: 1, + Report: &cos.BatchJobReport{ + Bucket: "qcs::cos:" + kBatchRegion + ":uid/" + s.Appid + ":" + kBatchBucket, + Enabled: "true", + Format: "Report_CSV_V1", + Prefix: "job-result", + ReportScope: "AllTasks", + }, + RoleArn: "qcs::cam::uin/" + kUin + ":roleName/COSBatch_QcsRole", + } + headers := &cos.BatchRequestHeaders{ + XCosAppid: kAppid, + } + + res1, _, err := client.Batch.CreateJob(context.Background(), opt, headers) + assert.Nil(s.T(), err, "create job Failed") + + jobid := res1.JobId + + res2, _, err := client.Batch.DescribeJob(context.Background(), jobid, headers) + assert.Nil(s.T(), err, "describe job Failed") + assert.Equal(s.T(), res2.Job.ConfirmationRequired, "true", "ConfirmationRequired not right") + assert.Equal(s.T(), res2.Job.Description, "test batch", "Description not right") + assert.Equal(s.T(), res2.Job.JobId, jobid, "jobid not right") + assert.Equal(s.T(), res2.Job.Priority, 1, "priority not right") + assert.Equal(s.T(), res2.Job.RoleArn, "qcs::cam::uin/"+kUin+":roleName/COSBatch_QcsRole", "priority not right") + + _, _, err = client.Batch.ListJobs(context.Background(), nil, headers) + assert.Nil(s.T(), err, "list jobs failed") + + up_opt := &cos.BatchUpdatePriorityOptions{ + JobId: jobid, + Priority: 3, + } + res3, _, err := client.Batch.UpdateJobPriority(context.Background(), up_opt, headers) + assert.Nil(s.T(), err, "list jobs failed") + assert.Equal(s.T(), res3.JobId, jobid, "jobid failed") + assert.Equal(s.T(), res3.Priority, 3, "priority not right") + + for i := 0; i < 10; i = i + 1 { + res, _, err := client.Batch.DescribeJob(context.Background(), jobid, headers) + assert.Nil(s.T(), err, "describe job Failed") + assert.Equal(s.T(), res2.Job.ConfirmationRequired, "true", "ConfirmationRequired not right") + assert.Equal(s.T(), res2.Job.Description, "test batch", "Description not right") + assert.Equal(s.T(), res2.Job.JobId, jobid, "jobid not right") + assert.Equal(s.T(), res2.Job.Priority, 1, "priority not right") + assert.Equal(s.T(), res2.Job.RoleArn, "qcs::cam::uin/"+kUin+":roleName/COSBatch_QcsRole", "priority not right") + if res.Job.Status == "Suspended" { + break + } + if i == 9 { + assert.Error(s.T(), errors.New("Job status is not Suspended or timeout")) + } + time.Sleep(time.Second * 2) + } + us_opt := &cos.BatchUpdateStatusOptions{ + JobId: jobid, + RequestedJobStatus: "Ready", // 允许状态转换见 https://cloud.tencent.com/document/product/436/38604 + StatusUpdateReason: "to test", + } + res4, _, err := client.Batch.UpdateJobStatus(context.Background(), us_opt, headers) + assert.Nil(s.T(), err, "list jobs failed") + assert.Equal(s.T(), res4.JobId, jobid, "jobid failed") + assert.Equal(s.T(), res4.Status, "Ready", "status failed") + assert.Equal(s.T(), res4.StatusUpdateReason, "to test", "StatusUpdateReason failed") +} + // End of api test // All methods that begin with "Test" are run as tests within a diff --git a/example/batch/create_job.go b/example/batch/create_job.go new file mode 100644 index 0000000..223f6bb --- /dev/null +++ b/example/batch/create_job.go @@ -0,0 +1,100 @@ +package main + +import ( + "context" + "fmt" + "net/http" + "net/url" + "os" + "strconv" + "strings" + + "github.com/google/uuid" + "github.com/tencentyun/cos-go-sdk-v5" + "github.com/tencentyun/cos-go-sdk-v5/debug" +) + +func main() { + test_batch_bucket := "testcd-1259654469" + target_batch_bucket := "cosgosdkreptest-1259654469" + appid := 1259654469 + uin := "100010805041" + region := "ap-chengdu" + + // bucket url:.cos..mycloud.com + bucketurl, _ := url.Parse("https://" + test_batch_bucket + ".cos." + region + ".myqcloud.com") + // batch url:.cos-control..myqcloud.ccom + batchurl, _ := url.Parse("https://" + uin + ".cos-control." + region + ".myqcloud.com") + + b := &cos.BaseURL{BucketURL: bucketurl, BatchURL: batchurl} + c := cos.NewClient(b, &http.Client{ + Transport: &cos.AuthorizationTransport{ + SecretID: os.Getenv("COS_SECRETID"), + SecretKey: os.Getenv("COS_SECRETKEY"), + Transport: &debug.DebugRequestTransport{ + RequestHeader: true, + RequestBody: true, + ResponseHeader: true, + ResponseBody: true, + }, + }, + }) + + // 创建需要复制的文件 + source_name := "test/1.txt" + sf := strings.NewReader("batch test content") + _, err := c.Object.Put(context.Background(), source_name, sf, nil) + if err != nil { + panic(err) + } + + // 创建清单文件 + manifest_name := "test/manifest.csv" + f := strings.NewReader(test_batch_bucket + "," + source_name) + resp, err := c.Object.Put(context.Background(), manifest_name, f, nil) + if err != nil { + panic(err) + } + etag := resp.Header.Get("ETag") + + uuid_str := uuid.New().String() + opt := &cos.BatchCreateJobOptions{ + ClientRequestToken: uuid_str, + ConfirmationRequired: "true", + Description: "test batch", + Manifest: &cos.BatchJobManifest{ + Location: &cos.BatchJobManifestLocation{ + ETag: etag, + ObjectArn: "qcs::cos:" + region + "::" + test_batch_bucket + "/" + manifest_name, + }, + Spec: &cos.BatchJobManifestSpec{ + Fields: []string{"Bucket", "Key"}, + Format: "COSBatchOperations_CSV_V1", + }, + }, + Operation: &cos.BatchJobOperation{ + PutObjectCopy: &cos.BatchJobOperationCopy{ + TargetResource: "qcs::cos:" + region + ":uid/" + strconv.Itoa(appid) + ":" + target_batch_bucket, + }, + }, + Priority: 1, + Report: &cos.BatchJobReport{ + Bucket: "qcs::cos:" + region + "::" + test_batch_bucket, + Enabled: "true", + Format: "Report_CSV_V1", + Prefix: "job-result", + ReportScope: "AllTasks", + }, + RoleArn: "qcs::cam::uin/" + uin + ":roleName/COSBatch_QcsRole", + } + headers := &cos.BatchRequestHeaders{ + XCosAppid: appid, + } + + res, _, err := c.Batch.CreateJob(context.Background(), opt, headers) + if err != nil { + panic(err) + } + fmt.Println(res) + +} diff --git a/example/batch/describe_job.go b/example/batch/describe_job.go new file mode 100644 index 0000000..6dff49d --- /dev/null +++ b/example/batch/describe_job.go @@ -0,0 +1,44 @@ +package main + +import ( + "context" + "net/http" + "net/url" + "os" + + "fmt" + "github.com/tencentyun/cos-go-sdk-v5" + "github.com/tencentyun/cos-go-sdk-v5/debug" +) + +func main() { + uin := "100010805041" + appid := 1259654469 + jobid := "795ad997-5557-4869-9a19-b66ec087d460" + u, _ := url.Parse("https://" + uin + ".cos-control.ap-chengdu.myqcloud.com") + b := &cos.BaseURL{BatchURL: u} + c := cos.NewClient(b, &http.Client{ + Transport: &cos.AuthorizationTransport{ + SecretID: os.Getenv("COS_SECRETID"), + SecretKey: os.Getenv("COS_SECRETKEY"), + Transport: &debug.DebugRequestTransport{ + RequestHeader: true, + RequestBody: true, + ResponseHeader: true, + ResponseBody: true, + }, + }, + }) + + headers := &cos.BatchRequestHeaders{ + XCosAppid: appid, + } + + res, _, err := c.Batch.DescribeJob(context.Background(), jobid, headers) + if err != nil { + panic(err) + } + if res != nil && res.Job != nil { + fmt.Printf("%+v", res.Job) + } +} diff --git a/example/batch/list_jobs.go b/example/batch/list_jobs.go new file mode 100644 index 0000000..b34b409 --- /dev/null +++ b/example/batch/list_jobs.go @@ -0,0 +1,43 @@ +package main + +import ( + "context" + "net/http" + "net/url" + "os" + + "fmt" + "github.com/tencentyun/cos-go-sdk-v5" + "github.com/tencentyun/cos-go-sdk-v5/debug" +) + +func main() { + uin := "100010805041" + appid := 1259654469 + u, _ := url.Parse("https://" + uin + ".cos-control.ap-chengdu.myqcloud.com") + b := &cos.BaseURL{BatchURL: u} + c := cos.NewClient(b, &http.Client{ + Transport: &cos.AuthorizationTransport{ + SecretID: os.Getenv("COS_SECRETID"), + SecretKey: os.Getenv("COS_SECRETKEY"), + Transport: &debug.DebugRequestTransport{ + RequestHeader: true, + RequestBody: true, + ResponseHeader: true, + ResponseBody: true, + }, + }, + }) + + headers := &cos.BatchRequestHeaders{ + XCosAppid: appid, + } + + res, _, err := c.Batch.ListJobs(context.Background(), nil, headers) + if err != nil { + panic(err) + } + if res != nil && res.Jobs != nil { + fmt.Printf("%+v", res.Jobs) + } +} diff --git a/example/batch/update_priority.go b/example/batch/update_priority.go new file mode 100644 index 0000000..eedf503 --- /dev/null +++ b/example/batch/update_priority.go @@ -0,0 +1,48 @@ +package main + +import ( + "context" + "net/http" + "net/url" + "os" + + "fmt" + "github.com/tencentyun/cos-go-sdk-v5" + "github.com/tencentyun/cos-go-sdk-v5/debug" +) + +func main() { + uin := "100010805041" + appid := 1259654469 + jobid := "795ad997-5557-4869-9a19-b66ec087d460" + u, _ := url.Parse("https://" + uin + ".cos-control.ap-chengdu.myqcloud.com") + b := &cos.BaseURL{BatchURL: u} + c := cos.NewClient(b, &http.Client{ + Transport: &cos.AuthorizationTransport{ + SecretID: os.Getenv("COS_SECRETID"), + SecretKey: os.Getenv("COS_SECRETKEY"), + Transport: &debug.DebugRequestTransport{ + RequestHeader: true, + RequestBody: true, + ResponseHeader: true, + ResponseBody: true, + }, + }, + }) + + opt := &cos.BatchUpdatePriorityOptions{ + JobId: jobid, + Priority: 3, + } + headers := &cos.BatchRequestHeaders{ + XCosAppid: appid, + } + + res, _, err := c.Batch.UpdateJobPriority(context.Background(), opt, headers) + if err != nil { + panic(err) + } + if res != nil { + fmt.Printf("%+v", res) + } +} diff --git a/example/batch/update_status.go b/example/batch/update_status.go new file mode 100644 index 0000000..f28130f --- /dev/null +++ b/example/batch/update_status.go @@ -0,0 +1,49 @@ +package main + +import ( + "context" + "net/http" + "net/url" + "os" + + "fmt" + "github.com/tencentyun/cos-go-sdk-v5" + "github.com/tencentyun/cos-go-sdk-v5/debug" +) + +func main() { + uin := "100010805041" + appid := 1259654469 + jobid := "289b0ea1-5ac5-453d-8a61-7f452dd4a209" + u, _ := url.Parse("https://" + uin + ".cos-control.ap-chengdu.myqcloud.com") + b := &cos.BaseURL{BatchURL: u} + c := cos.NewClient(b, &http.Client{ + Transport: &cos.AuthorizationTransport{ + SecretID: os.Getenv("COS_SECRETID"), + SecretKey: os.Getenv("COS_SECRETKEY"), + Transport: &debug.DebugRequestTransport{ + RequestHeader: true, + RequestBody: true, + ResponseHeader: true, + ResponseBody: true, + }, + }, + }) + + opt := &cos.BatchUpdateStatusOptions{ + JobId: jobid, + RequestedJobStatus: "Ready", // 允许状态转换见 https://cloud.tencent.com/document/product/436/38604 + StatusUpdateReason: "to test", + } + headers := &cos.BatchRequestHeaders{ + XCosAppid: appid, + } + + res, _, err := c.Batch.UpdateJobStatus(context.Background(), opt, headers) + if err != nil { + panic(err) + } + if res != nil { + fmt.Printf("%+v", res) + } +} diff --git a/go.mod b/go.mod index 35afc5c..c01628b 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.12 require ( github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409 github.com/google/go-querystring v1.0.0 + github.com/google/uuid v1.1.1 github.com/mozillazg/go-httpheader v0.2.1 github.com/stretchr/testify v1.3.0 ) diff --git a/go.sum b/go.sum index 5ee5a1d..ab8a08d 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,8 @@ github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/mozillazg/go-httpheader v0.2.1 h1:geV7TrjbL8KXSyvghnFm+NyTux/hxwueTSrwhe88TQQ= github.com/mozillazg/go-httpheader v0.2.1/go.mod h1:jJ8xECTlalr6ValeXYdOF8fFUISeBAdw6E61aqQma60= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=