tommy
/
cos-go-sdk-v5
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

add batch

tags/v0.7.8
jojoliang 5 years ago
parent
c807d39e90
commit
c57ac81a16
10 changed files with 1070 additions and 11 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 262
      batch.go
  2. 389
      batch_test.go
  3. 5
      cos.go
  4. 2
      cos_test.go
  5. 138
      costesting/ci_test.go
  6. 99
      example/batch/create_job.go
  7. 44
      example/batch/describe_job.go
  8. 43
      example/batch/list_jobs.go
  9. 48
      example/batch/update_priority.go
  10. 49
      example/batch/update_status.go

262
batch.go
View File

@ -0,0 +1,262 @@
package cos
import (
"context"
"encoding/xml"
"fmt"
"net/http"
)
type BatchService service
type BatchRequestHeaders struct {
XCosAppid int `header:"x-cos-appid"`
ContentLength string `header:"Content-Length,omitempty"`
ContentType string `header:"Content-Type,omitempty"`
Headers *http.Header `header:"-"`
}
// BatchProgressSummary
type BatchProgressSummary struct {
NumberOfTasksFailed int `xml:"NumberOfTasksFailed"`
NumberOfTasksSucceeded int `xml:"NumberOfTasksSucceeded"`
TotalNumberOfTasks int `xml:"TotalNumberOfTasks"`
}
// BatchJobReport
type BatchJobReport struct {
Bucket string `xml:"Bucket"`
Enabled string `xml:"Enabled"`
Format string `xml:"Format"`
Prefix string `xml:"Prefix,omitempty"`
ReportScope string `xml:"ReportScope"`
}
// BatchJobOperationCopy
type BatchMetadata struct {
Key string `xml:"Key"`
Value string `xml:"Value"`
}
type BatchNewObjectMetadata struct {
CacheControl string `xml:"CacheControl,omitempty"`
ContentDisposition string `xml:"ContentDisposition,omitempty"`
ContentEncoding string `xml:"ContentEncoding,omitempty"`
ContentType string `xml:"ContentType,omitempty"`
HttpExpiresDate string `xml:"HttpExpiresDate,omitempty"`
SSEAlgorithm string `xml:"SSEAlgorithm,omitempty"`
UserMetadata []BatchMetadata `xml:"UserMetadata>member,omitempty"`
}
type BatchGrantee struct {
DisplayName string `xml:"DisplayName,omitempty"`
Identifier string `xml:"Identifier"`
TypeIdentifier string `xml:"TypeIdentifier"`
}
type BatchCOSGrant struct {
Grantee *BatchGrantee `xml:"Grantee"`
Permission string `xml:"Permission"`
}
type BatchAccessControlGrants struct {
COSGrants *BatchCOSGrant `xml:"COSGrant,omitempty"`
}
type BatchJobOperationCopy struct {
AccessControlGrants *BatchAccessControlGrants `xml:"AccessControlGrants,omitempty"`
CannedAccessControlList string `xml:"CannedAccessControlList,omitempty"`
MetadataDirective string `xml:"MetadataDirective,omitempty"`
ModifiedSinceConstraint int64 `xml:"ModifiedSinceConstraint,omitempty"`
UnModifiedSinceConstraint int64 `xml:"UnModifiedSinceConstraint,omitempty"`
NewObjectMetadata *BatchNewObjectMetadata `xml:"NewObjectMetadata,omitempty"`
StorageClass string `xml:"StorageClass,omitempty"`
TargetResource string `xml:"TargetResource"`
}
// BatchJobOperation
type BatchJobOperation struct {
PutObjectCopy *BatchJobOperationCopy `xml:"COSPutObjectCopy,omitempty" header:"-"`
}
// BatchJobManifest
type BatchJobManifestLocation struct {
ETag string `xml:"ETag" header:"-"`
ObjectArn string `xml:"ObjectArn" header:"-"`
ObjectVersionId string `xml:"ObjectVersionId,omitempty" header:"-"`
}
type BatchJobManifestSpec struct {
Fields []string `xml:"Fields>member,omitempty" header:"-"`
Format string `xml:"Format" header:"-"`
}
type BatchJobManifest struct {
Location *BatchJobManifestLocation `xml:"Location" header:"-"`
Spec *BatchJobManifestSpec `xml:"Spec" header:"-"`
}
type BatchCreateJobOptions struct {
XMLName xml.Name `xml:"CreateJobRequest" header:"-"`
ClientRequestToken string `xml:"ClientRequestToken" header:"-"`
ConfirmationRequired string `xml:"ConfirmationRequired,omitempty" header:"-"`
Description string `xml:"Description,omitempty" header:"-"`
Manifest *BatchJobManifest `xml:"Manifest" header:"-"`
Operation *BatchJobOperation `xml:"Operation" header:"-"`
Priority int `xml:"Priority" header:"-"`
Report *BatchJobReport `xml:"Report" header:"-"`
RoleArn string `xml:"RoleArn" header:"-"`
}
type BatchCreateJobResult struct {
XMLName xml.Name `xml:"CreateJobResult"`
JobId string `xml:"JobId"`
}
func processETag(opt *BatchCreateJobOptions) *BatchCreateJobOptions {
if opt != nil && opt.Manifest != nil && opt.Manifest.Location != nil {
opt.Manifest.Location.ETag = "<ETag>" + opt.Manifest.Location.ETag + "</ETag>"
}
return opt
}
func (s *BatchService) CreateJob(ctx context.Context, opt *BatchCreateJobOptions, headers *BatchRequestHeaders) (*BatchCreateJobResult, *Response, error) {
var res BatchCreateJobResult
sendOpt := sendOptions{
baseURL: s.client.BaseURL.BatchURL,
uri: "/jobs",
method: http.MethodPost,
optHeader: headers,
body: opt,
result: &res,
}
resp, err := s.client.send(ctx, &sendOpt)
return &res, resp, err
}
type BatchJobFailureReasons struct {
FailureCode string `xml:"FailureCode"`
FailureReason string `xml:"FailureReason"`
}
type BatchDescribeJob struct {
ConfirmationRequired string `xml:"ConfirmationRequired,omitempty"`
CreationTime string `xml:"CreationTime,omitempty"`
Description string `xml:"Description,omitempty"`
FailureReasons *BatchJobFailureReasons `xml:"FailureReasons>JobFailure,omitempty"`
JobId string `xml:"JobId"`
Manifest *BatchJobManifest `xml:"Manifest"`
Operation *BatchJobOperation `xml:"Operation"`
Priority int `xml:"Priority"`
ProgressSummary *BatchProgressSummary `xml:"ProgressSummary"`
Report *BatchJobReport `xml:"Report,omitempty"`
RoleArn string `xml:"RoleArn,omitempty"`
Status string `xml:"Status,omitempty"`
StatusUpdateReason string `xml:"StatusUpdateReason,omitempty"`
SuspendedCause string `xml:"SuspendedCause,omitempty"`
SuspendedDate string `xml:"SuspendedDate,omitempty"`
TerminationDate string `xml:"TerminationDate,omitempty"`
}
type BatchDescribeJobResult struct {
XMLName xml.Name `xml:"DescribeJobResult"`
Job *BatchDescribeJob `xml:"Job"`
}
func (s *BatchService) DescribeJob(ctx context.Context, id string, headers *BatchRequestHeaders) (*BatchDescribeJobResult, *Response, error) {
var res BatchDescribeJobResult
u := fmt.Sprintf("/jobs/%s", id)
sendOpt := sendOptions{
baseURL: s.client.BaseURL.BatchURL,
uri: u,
method: http.MethodGet,
optHeader: headers,
result: &res,
}
resp, err := s.client.send(ctx, &sendOpt)
return &res, resp, err
}
type BatchListJobsOptions struct {
JobStatuses string `url:"jobStatuses,omitempty" header:"-"`
MaxResults int `url:"maxResults,omitempty" header:"-"`
NextToken string `url:"nextToken,omitempty" header:"-"`
}
type BatchListJobsMember struct {
CreationTime string `xml:"CreationTime,omitempty"`
Description string `xml:"Description,omitempty"`
JobId string `xml:"JobId,omitempty"`
Operation string `xml:"Operation,omitempty"`
Priority int `xml:"Priority,omitempty"`
ProgressSummary *BatchProgressSummary `xml:"ProgressSummary,omitempty"`
Status string `xml:"Status,omitempty"`
TerminationDate string `xml:"TerminationDate,omitempty"`
}
type BatchListJobs struct {
Members []BatchListJobsMember `xml:"member,omitempty"`
}
type BatchListJobsResult struct {
XMLName xml.Name `xml:"ListJobsResult"`
Jobs *BatchListJobs `xml:"Jobs"`
NextToken string `xml:"NextToken,omitempty"`
}
func (s *BatchService) ListJobs(ctx context.Context, opt *BatchListJobsOptions, headers *BatchRequestHeaders) (*BatchListJobsResult, *Response, error) {
var res BatchListJobsResult
sendOpt := sendOptions{
baseURL: s.client.BaseURL.BatchURL,
uri: "/jobs",
method: http.MethodGet,
optQuery: opt,
optHeader: headers,
result: &res,
}
resp, err := s.client.send(ctx, &sendOpt)
return &res, resp, err
}
type BatchUpdatePriorityOptions struct {
JobId string `url:"-" header:"-" xml:"-"`
Priority int `url:"priority" header:"-" xml:"-"`
}
type BatchUpdatePriorityResult struct {
XMLName xml.Name `xml:"UpdateJobPriorityResult"`
JobId string `xml:"JobId,omitempty"`
Priority int `xml:"Priority,omitempty"`
}
func (s *BatchService) UpdateJobPriority(ctx context.Context, opt *BatchUpdatePriorityOptions, headers *BatchRequestHeaders) (*BatchUpdatePriorityResult, *Response, error) {
u := fmt.Sprintf("/jobs/%s/priority", opt.JobId)
var res BatchUpdatePriorityResult
sendOpt := sendOptions{
baseURL: s.client.BaseURL.BatchURL,
uri: u,
method: http.MethodPost,
optQuery: opt,
optHeader: headers,
result: &res,
}
resp, err := s.client.send(ctx, &sendOpt)
return &res, resp, err
}
type BatchUpdateStatusOptions struct {
JobId string `header:"-" url:"-" xml:"-"`
RequestedJobStatus string `url:"requestedJobStatus" header:"-" xml:"-"`
StatusUpdateReason string `url:"statusUpdateReason,omitempty" header:"-", xml:"-"`
}
type BatchUpdateStatusResult struct {
XMLName xml.Name `xml:"UpdateJobStatusResult"`
JobId string `xml:"JobId,omitempty"`
Status string `xml:"Status,omitempty"`
StatusUpdateReason string `xml:"StatusUpdateReason,omitempty"`
}
func (s *BatchService) UpdateJobStatus(ctx context.Context, opt *BatchUpdateStatusOptions, headers *BatchRequestHeaders) (*BatchUpdateStatusResult, *Response, error) {
u := fmt.Sprintf("/jobs/%s/status", opt.JobId)
var res BatchUpdateStatusResult
sendOpt := sendOptions{
baseURL: s.client.BaseURL.BatchURL,
uri: u,
method: http.MethodPost,
optQuery: opt,
optHeader: headers,
result: &res,
}
resp, err := s.client.send(ctx, &sendOpt)
return &res, resp, err
}

389
batch_test.go
View File

@ -0,0 +1,389 @@
package cos
import (
"context"
"encoding/xml"
"fmt"
"net/http"
"reflect"
"testing"
"github.com/google/uuid"
)
func TestBatchService_CreateJob(t *testing.T) {
setup()
defer teardown()
uuid_str := uuid.New().String()
opt := &BatchCreateJobOptions{
ClientRequestToken: uuid_str,
Description: "test batch",
Manifest: &BatchJobManifest{
Location: &BatchJobManifestLocation{
ETag: "15150651828fa9cdcb8356b6d1c7638b",
ObjectArn: "qcs::cos:ap-chengdu::sourcebucket-1250000000/manifests/batch-copy-manifest.csv",
},
Spec: &BatchJobManifestSpec{
Fields: []string{"Bucket", "Key"},
Format: "COSBatchOperations_CSV_V1",
},
},
Operation: &BatchJobOperation{
PutObjectCopy: &BatchJobOperationCopy{
TargetResource: "qcs::cos:ap-chengdu::destinationbucket-1250000000",
},
},
Priority: 1,
Report: &BatchJobReport{
Bucket: "qcs::cos:ap-chengdu::sourcebucket-1250000000",
Enabled: "true",
Format: "Report_CSV_V1",
Prefix: "job-result",
ReportScope: "AllTasks",
},
RoleArn: "qcs::cam::uin/100000000001:roleName/COS_Batch_QcsRole",
}
mux.HandleFunc("/jobs", func(w http.ResponseWriter, r *http.Request) {
testHeader(t, r, "x-cos-appid", "1250000000")
testMethod(t, r, http.MethodPost)
v := new(BatchCreateJobOptions)
xml.NewDecoder(r.Body).Decode(v)
want := opt
want.XMLName = xml.Name{Local: "CreateJobRequest"}
if !reflect.DeepEqual(v, want) {
t.Errorf("Batch.CreateJob request body: %+v, want %+v", v, want)
}
fmt.Fprint(w, `<?xml version='1.0' encoding='utf-8' ?>
<CreateJobResult>
<JobId>53dc6228-c50b-46f7-8ad7-65e7159f1aae</JobId>
</CreateJobResult>`)
})
headers := &BatchRequestHeaders{
XCosAppid: 1250000000,
}
ref, _, err := client.Batch.CreateJob(context.Background(), opt, headers)
if err != nil {
t.Fatalf("Batch.CreateJob returned error: %v", err)
}
want := &BatchCreateJobResult{
XMLName: xml.Name{Local: "CreateJobResult"},
JobId: "53dc6228-c50b-46f7-8ad7-65e7159f1aae",
}
if !reflect.DeepEqual(ref, want) {
t.Errorf("Batch.CreateJob returned %+v, want %+v", ref, want)
}
}
func TestBatchService_DescribeJob(t *testing.T) {
setup()
defer teardown()
mux.HandleFunc("/jobs/53dc6228-c50b-46f7-8ad7-65e7159f1aae", func(w http.ResponseWriter, r *http.Request) {
testHeader(t, r, "x-cos-appid", "1250000000")
testMethod(t, r, http.MethodGet)
fmt.Fprint(w, `<?xml version='1.0' encoding='utf-8' ?>
<DescribeJobResult>
<Job>
<ConfirmationRequired>false</ConfirmationRequired>
<CreationTime>2019-12-19T18:00:30Z</CreationTime>
<Description>example-job</Description>
<FailureReasons>
<JobFailure>
<FailureCode/>
<FailureReason/>
</JobFailure>
</FailureReasons>
<JobId>53dc6228-c50b-46f7-8ad7-65e7159f1aae</JobId>
<Manifest>
<Location>
<ETag>&quot;15150651828fa9cdcb8356b6d1c7638b&quot;</ETag>
<ObjectArn>qcs::cos:ap-chengdu::sourcebucket-1250000000/manifests/batch-copy-manifest.csv</ObjectArn>
</Location>
<Spec>
<Fields>
<member>Bucket</member>
<member>Key</member>
</Fields>
<Format>COSBatchOperations_CSV_V1</Format>
</Spec>
</Manifest>
<Operation>
<COSPutObjectCopy>
<TargetResource>qcs::cos:ap-chengdu::destinationbucket-1250000000</TargetResource>
</COSPutObjectCopy>
</Operation>
<Priority>10</Priority>
<ProgressSummary>
<NumberOfTasksFailed>0</NumberOfTasksFailed>
<NumberOfTasksSucceeded>10</NumberOfTasksSucceeded>
<TotalNumberOfTasks>10</TotalNumberOfTasks>
</ProgressSummary>
<Report>
<Bucket>qcs::cos:ap-chengdu::sourcebucket-1250000000</Bucket>
<Enabled>true</Enabled>
<Format>Report_CSV_V1</Format>
<Prefix>job-result</Prefix>
<ReportScope>AllTasks</ReportScope>
</Report>
<RoleArn>qcs::cam::uin/100000000001:roleName/COS_Batch_QcsRole</RoleArn>
<Status>Complete</Status>
<StatusUpdateReason>Job complete</StatusUpdateReason>
<TerminationDate>2019-12-19T18:00:42Z</TerminationDate>
</Job>
</DescribeJobResult>`)
})
headers := &BatchRequestHeaders{
XCosAppid: 1250000000,
}
ref, _, err := client.Batch.DescribeJob(context.Background(), "53dc6228-c50b-46f7-8ad7-65e7159f1aae", headers)
if err != nil {
t.Fatalf("Batch.DescribeJob returned error: %v", err)
}
want := &BatchDescribeJobResult{
XMLName: xml.Name{Local: "DescribeJobResult"},
Job: &BatchDescribeJob{
ConfirmationRequired: "false",
CreationTime: "2019-12-19T18:00:30Z",
Description: "example-job",
FailureReasons: &BatchJobFailureReasons{},
JobId: "53dc6228-c50b-46f7-8ad7-65e7159f1aae",
Manifest: &BatchJobManifest{
Location: &BatchJobManifestLocation{
ETag: "\"15150651828fa9cdcb8356b6d1c7638b\"",
ObjectArn: "qcs::cos:ap-chengdu::sourcebucket-1250000000/manifests/batch-copy-manifest.csv",
},
Spec: &BatchJobManifestSpec{
Fields: []string{"Bucket", "Key"},
Format: "COSBatchOperations_CSV_V1",
},
},
Operation: &BatchJobOperation{
PutObjectCopy: &BatchJobOperationCopy{
TargetResource: "qcs::cos:ap-chengdu::destinationbucket-1250000000",
},
},
Priority: 10,
ProgressSummary: &BatchProgressSummary{
NumberOfTasksFailed: 0,
NumberOfTasksSucceeded: 10,
TotalNumberOfTasks: 10,
},
Report: &BatchJobReport{
Bucket: "qcs::cos:ap-chengdu::sourcebucket-1250000000",
Enabled: "true",
Format: "Report_CSV_V1",
Prefix: "job-result",
ReportScope: "AllTasks",
},
RoleArn: "qcs::cam::uin/100000000001:roleName/COS_Batch_QcsRole",
Status: "Complete",
StatusUpdateReason: "Job complete",
TerminationDate: "2019-12-19T18:00:42Z",
},
}
if !reflect.DeepEqual(ref, want) {
t.Errorf("Batch.DescribeJob returned %+v, want %+v", ref, want)
}
}
func TestBatchService_ListJobs(t *testing.T) {
setup()
defer teardown()
mux.HandleFunc("/jobs", func(w http.ResponseWriter, r *http.Request) {
testHeader(t, r, "x-cos-appid", "1250000000")
testMethod(t, r, http.MethodGet)
vs := values{
"maxResults": "2",
}
testFormValues(t, r, vs)
fmt.Fprint(w, `<?xml version='1.0' encoding='utf-8' ?>
<ListJobsResult>
<Jobs>
<member>
<CreationTime>2019-12-19T11:05:40Z</CreationTime>
<Description>example-job</Description>
<JobId>021140d8-67ca-4e89-8089-0de9a1e40943</JobId>
<Operation>COSPutObjectCopy</Operation>
<Priority>10</Priority>
<ProgressSummary>
<NumberOfTasksFailed>0</NumberOfTasksFailed>
<NumberOfTasksSucceeded>10</NumberOfTasksSucceeded>
<TotalNumberOfTasks>10</TotalNumberOfTasks>
</ProgressSummary>
<Status>Complete</Status>
<TerminationDate>2019-12-19T11:05:56Z</TerminationDate>
</member>
<member>
<CreationTime>2019-12-19T11:07:05Z</CreationTime>
<Description>example-job</Description>
<JobId>066d919e-49b9-429e-b844-e17ea7b16421</JobId>
<Operation>COSPutObjectCopy</Operation>
<Priority>10</Priority>
<ProgressSummary>
<NumberOfTasksFailed>0</NumberOfTasksFailed>
<NumberOfTasksSucceeded>10</NumberOfTasksSucceeded>
<TotalNumberOfTasks>10</TotalNumberOfTasks>
</ProgressSummary>
<Status>Complete</Status>
<TerminationDate>2019-12-19T11:07:21Z</TerminationDate>
</member>
</Jobs>
<NextToken>066d919e-49b9-429e-b844-e17ea7b16421</NextToken>
</ListJobsResult>`)
})
opt := &BatchListJobsOptions{
MaxResults: 2,
}
headers := &BatchRequestHeaders{
XCosAppid: 1250000000,
}
ref, _, err := client.Batch.ListJobs(context.Background(), opt, headers)
if err != nil {
t.Fatalf("Batch.DescribeJob returned error: %v", err)
}
want := &BatchListJobsResult{
XMLName: xml.Name{Local: "ListJobsResult"},
Jobs: &BatchListJobs{
Members: []BatchListJobsMember{
{
CreationTime: "2019-12-19T11:05:40Z",
Description: "example-job",
JobId: "021140d8-67ca-4e89-8089-0de9a1e40943",
Operation: "COSPutObjectCopy",
Priority: 10,
ProgressSummary: &BatchProgressSummary{
NumberOfTasksFailed: 0,
NumberOfTasksSucceeded: 10,
TotalNumberOfTasks: 10,
},
Status: "Complete",
TerminationDate: "2019-12-19T11:05:56Z",
},
{
CreationTime: "2019-12-19T11:07:05Z",
Description: "example-job",
JobId: "066d919e-49b9-429e-b844-e17ea7b16421",
Operation: "COSPutObjectCopy",
Priority: 10,
ProgressSummary: &BatchProgressSummary{
NumberOfTasksFailed: 0,
NumberOfTasksSucceeded: 10,
TotalNumberOfTasks: 10,
},
Status: "Complete",
TerminationDate: "2019-12-19T11:07:21Z",
},
},
},
NextToken: "066d919e-49b9-429e-b844-e17ea7b16421",
}
if !reflect.DeepEqual(ref, want) {
t.Errorf("Batch.ListJobs returned %+v, want %+v", ref, want)
}
}
func TestBatchService_UpdateJobsPriority(t *testing.T) {
setup()
defer teardown()
mux.HandleFunc("/jobs/021140d8-67ca-4e89-8089-0de9a1e40943/priority", func(w http.ResponseWriter, r *http.Request) {
testHeader(t, r, "x-cos-appid", "1250000000")
testMethod(t, r, http.MethodPost)
vs := values{
"priority": "10",
}
testFormValues(t, r, vs)
fmt.Fprint(w, `<?xml version='1.0' encoding='utf-8' ?>
<UpdateJobPriorityResult>
<JobId>021140d8-67ca-4e89-8089-0de9a1e40943</JobId>
<Priority>10</Priority>
</UpdateJobPriorityResult>`)
})
opt := &BatchUpdatePriorityOptions{
JobId: "021140d8-67ca-4e89-8089-0de9a1e40943",
Priority: 10,
}
headers := &BatchRequestHeaders{
XCosAppid: 1250000000,
}
ref, _, err := client.Batch.UpdateJobPriority(context.Background(), opt, headers)
if err != nil {
t.Fatalf("Batch.UpdateJobPriority returned error: %v", err)
}
want := &BatchUpdatePriorityResult{
XMLName: xml.Name{Local: "UpdateJobPriorityResult"},
JobId: "021140d8-67ca-4e89-8089-0de9a1e40943",
Priority: 10,
}
if !reflect.DeepEqual(ref, want) {
t.Errorf("Batch.UpdateJobsPriority returned %+v, want %+v", ref, want)
}
}
func TestBatchService_UpdateJobsStatus(t *testing.T) {
setup()
defer teardown()
mux.HandleFunc("/jobs/021140d8-67ca-4e89-8089-0de9a1e40943/status", func(w http.ResponseWriter, r *http.Request) {
testHeader(t, r, "x-cos-appid", "1250000000")
testMethod(t, r, http.MethodPost)
vs := values{
"requestedJobStatus": "Ready",
"statusUpdateReason": "to do",
}
testFormValues(t, r, vs)
fmt.Fprint(w, `<?xml version='1.0' encoding='utf-8' ?>
<UpdateJobStatusResult>
<JobId>021140d8-67ca-4e89-8089-0de9a1e40943</JobId>
<Status>Ready</Status>
<StatusUpdateReason>to do</StatusUpdateReason>
</UpdateJobStatusResult>`)
})
opt := &BatchUpdateStatusOptions{
JobId: "021140d8-67ca-4e89-8089-0de9a1e40943",
RequestedJobStatus: "Ready",
StatusUpdateReason: "to do",
}
headers := &BatchRequestHeaders{
XCosAppid: 1250000000,
}
ref, _, err := client.Batch.UpdateJobStatus(context.Background(), opt, headers)
if err != nil {
t.Fatalf("Batch.UpdateJobStatus returned error: %v", err)
}
want := &BatchUpdateStatusResult{
XMLName: xml.Name{Local: "UpdateJobStatusResult"},
JobId: "021140d8-67ca-4e89-8089-0de9a1e40943",
Status: "Ready",
StatusUpdateReason: "to do",
}
if !reflect.DeepEqual(ref, want) {
t.Errorf("Batch.UpdateJobsStatus returned %+v, want %+v", ref, want)
}
}

5
cos.go
View File

@ -39,6 +39,8 @@ type BaseURL struct {
BucketURL *url.URL BucketURL *url.URL
// 访问 service API 的基础 URL（不包含 path 部分）: http://example.com // 访问 service API 的基础 URL（不包含 path 部分）: http://example.com
ServiceURL *url.URL ServiceURL *url.URL
// 访问 job API 的基础 URL （不包含 path 部分）: http://example.com
BatchURL *url.URL
} }
// NewBucketURL 生成 BaseURL 所需的 BucketURL // NewBucketURL 生成 BaseURL 所需的 BucketURL
@ -78,6 +80,7 @@ type Client struct {
Service *ServiceService Service *ServiceService
Bucket *BucketService Bucket *BucketService
Object *ObjectService Object *ObjectService
Batch *BatchService
} }
type service struct { type service struct {
@ -94,6 +97,7 @@ func NewClient(uri *BaseURL, httpClient *http.Client) *Client {
if uri != nil { if uri != nil {
baseURL.BucketURL = uri.BucketURL baseURL.BucketURL = uri.BucketURL
baseURL.ServiceURL = uri.ServiceURL baseURL.ServiceURL = uri.ServiceURL
baseURL.BatchURL = uri.BatchURL
} }
if baseURL.ServiceURL == nil { if baseURL.ServiceURL == nil {
baseURL.ServiceURL, _ = url.Parse(defaultServiceBaseURL) baseURL.ServiceURL, _ = url.Parse(defaultServiceBaseURL)
@ -108,6 +112,7 @@ func NewClient(uri *BaseURL, httpClient *http.Client) *Client {
c.Service = (*ServiceService)(&c.common) c.Service = (*ServiceService)(&c.common)
c.Bucket = (*BucketService)(&c.common) c.Bucket = (*BucketService)(&c.common)
c.Object = (*ObjectService)(&c.common) c.Object = (*ObjectService)(&c.common)
c.Batch = (*BatchService)(&c.common)
return c return c
} }

2
cos_test.go
View File

@ -32,7 +32,7 @@ func setup() {
server = httptest.NewServer(mux) server = httptest.NewServer(mux)
u, _ := url.Parse(server.URL) u, _ := url.Parse(server.URL)
client = NewClient(&BaseURL{u, u}, nil)
client = NewClient(&BaseURL{u, u, u}, nil)
} }
// teardown closes the test HTTP server. // teardown closes the test HTTP server.

138
costesting/ci_test.go
View File

@ -3,6 +3,7 @@ package cos
// Basic imports // Basic imports
import ( import (
"context" "context"
"errors"
"fmt" "fmt"
"io/ioutil" "io/ioutil"
"math/rand" "math/rand"
@ -13,6 +14,7 @@ import (
"testing" "testing"
"time" "time"
"github.com/google/uuid"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite" "github.com/stretchr/testify/suite"
"github.com/tencentyun/cos-go-sdk-v5" "github.com/tencentyun/cos-go-sdk-v5"
@ -44,14 +46,24 @@ type CosTestSuite struct {
SepFileName string SepFileName string
} }
// replace
// 请替换成您的账号及存储桶信息
const ( const (
//uin //uin
kUin = "100010805041" kUin = "100010805041"
//Replication Target Region,
kRepRegion = "ap-chengdu"
//Replication Target Bucket, Version management needs to be turned on beforehand
kAppid = 1259654469
// 常规测试需要的存储桶
kBucket = "cosgosdktest-1259654469"
kRegion = "ap-guangzhou"
// 跨区域复制需要的目标存储桶，地域不能与kBucket存储桶相同。
kRepBucket = "cosgosdkreptest" kRepBucket = "cosgosdkreptest"
kRepRegion = "ap-chengdu"
// Batch测试需要的源存储桶和目标存储桶，目前只在成都、重庆地域公测
kBatchBucket = "testcd-1259654469"
kTargetBatchBucket = "cosgosdkreptest-1259654469" //复用了存储桶
kBatchRegion = "ap-chengdu"
) )
func (s *CosTestSuite) SetupSuite() { func (s *CosTestSuite) SetupSuite() {
@ -62,11 +74,13 @@ func (s *CosTestSuite) SetupSuite() {
// CI client for test interface // CI client for test interface
// URL like this http://test-1253846586.cos.ap-guangzhou.myqcloud.com // URL like this http://test-1253846586.cos.ap-guangzhou.myqcloud.com
u := "https://cosgosdktest-1259654469.cos.ap-guangzhou.myqcloud.com"
u := "https://" + kBucket + ".cos." + kRegion + ".myqcloud.com"
u2 := "https://" + kUin + ".cos-control." + kBatchRegion + ".myqcloud.com"
// Get the region // Get the region
iu, _ := url.Parse(u)
p := strings.Split(iu.Host, ".")
bucketurl, _ := url.Parse(u)
batchurl, _ := url.Parse(u2)
p := strings.Split(bucketurl.Host, ".")
assert.Equal(s.T(), 5, len(p), "Bucket host is not right") assert.Equal(s.T(), 5, len(p), "Bucket host is not right")
s.Region = p[2] s.Region = p[2]
@ -75,7 +89,7 @@ func (s *CosTestSuite) SetupSuite() {
s.Bucket = pp[0] s.Bucket = pp[0]
s.Appid = pp[1] s.Appid = pp[1]
ib := &cos.BaseURL{BucketURL: iu}
ib := &cos.BaseURL{BucketURL: bucketurl, BatchURL: batchurl}
s.Client = cos.NewClient(ib, &http.Client{ s.Client = cos.NewClient(ib, &http.Client{
Transport: &cos.AuthorizationTransport{ Transport: &cos.AuthorizationTransport{
SecretID: os.Getenv("COS_SECRETID"), SecretID: os.Getenv("COS_SECRETID"),
@ -105,7 +119,7 @@ func (s *CosTestSuite) TestGetService() {
// Bucket API // Bucket API
func (s *CosTestSuite) TestPutHeadDeleteBucket() { func (s *CosTestSuite) TestPutHeadDeleteBucket() {
// Notic sometimes the bucket host can not analyis, may has i/o timeout problem // Notic sometimes the bucket host can not analyis, may has i/o timeout problem
u := "http://gosdkbuckettest-" + s.Appid + ".cos.ap-beijing-1.myqcloud.com"
u := "http://" + "testgosdkbucket-create-head-del-" + s.Appid + ".cos." + kRegion + ".myqcloud.com"
iu, _ := url.Parse(u) iu, _ := url.Parse(u)
ib := &cos.BaseURL{BucketURL: iu} ib := &cos.BaseURL{BucketURL: iu}
client := cos.NewClient(ib, &http.Client{ client := cos.NewClient(ib, &http.Client{
@ -710,6 +724,112 @@ func (s *CosTestSuite) TestMultiUpload() {
assert.Nil(s.T(), err, "remove tmp file failed") assert.Nil(s.T(), err, "remove tmp file failed")
} }
func (s *CosTestSuite) TestBatch() {
client := cos.NewClient(s.Client.BaseURL, &http.Client{
Transport: &cos.AuthorizationTransport{
SecretID: os.Getenv("COS_SECRETID"),
SecretKey: os.Getenv("COS_SECRETKEY"),
},
})
source_name := "test/1.txt"
sf := strings.NewReader("batch test content")
_, err := client.Object.Put(context.Background(), source_name, sf, nil)
assert.Nil(s.T(), err, "object put Failed")
manifest_name := "test/manifest.csv"
f := strings.NewReader(kBatchBucket + "," + source_name)
resp, err := client.Object.Put(context.Background(), manifest_name, f, nil)
assert.Nil(s.T(), err, "object put Failed")
etag := resp.Header.Get("ETag")
uuid_str := uuid.New().String()
opt := &cos.BatchCreateJobOptions{
ClientRequestToken: uuid_str,
ConfirmationRequired: "true",
Description: "test batch",
Manifest: &cos.BatchJobManifest{
Location: &cos.BatchJobManifestLocation{
ETag: etag,
ObjectArn: "qcs::cos:" + kBatchRegion + "::" + kBatchBucket + "/" + manifest_name,
},
Spec: &cos.BatchJobManifestSpec{
Fields: []string{"Bucket", "Key"},
Format: "COSBatchOperations_CSV_V1",
},
},
Operation: &cos.BatchJobOperation{
PutObjectCopy: &cos.BatchJobOperationCopy{
TargetResource: "qcs::cos:" + kBatchRegion + "::" + kTargetBatchBucket,
},
},
Priority: 1,
Report: &cos.BatchJobReport{
Bucket: "qcs::cos:" + kBatchRegion + "::" + kBatchBucket,
Enabled: "true",
Format: "Report_CSV_V1",
Prefix: "job-result",
ReportScope: "AllTasks",
},
RoleArn: "qcs::cam::uin/" + kUin + ":roleName/COSBatch_QcsRole",
}
headers := &cos.BatchRequestHeaders{
XCosAppid: kAppid,
}
res1, _, err := client.Batch.CreateJob(context.Background(), opt, headers)
assert.Nil(s.T(), err, "create job Failed")
jobid := res1.JobId
res2, _, err := client.Batch.DescribeJob(context.Background(), jobid, headers)
assert.Nil(s.T(), err, "describe job Failed")
assert.Equal(s.T(), res2.Job.ConfirmationRequired, "true", "ConfirmationRequired not right")
assert.Equal(s.T(), res2.Job.Description, "test batch", "Description not right")
assert.Equal(s.T(), res2.Job.JobId, jobid, "jobid not right")
assert.Equal(s.T(), res2.Job.Priority, 1, "priority not right")
assert.Equal(s.T(), res2.Job.RoleArn, "qcs::cam::uin/"+kUin+":roleName/COSBatch_QcsRole", "priority not right")
_, _, err = client.Batch.ListJobs(context.Background(), nil, headers)
assert.Nil(s.T(), err, "list jobs failed")
up_opt := &cos.BatchUpdatePriorityOptions{
JobId: jobid,
Priority: 3,
}
res3, _, err := client.Batch.UpdateJobPriority(context.Background(), up_opt, headers)
assert.Nil(s.T(), err, "list jobs failed")
assert.Equal(s.T(), res3.JobId, jobid, "jobid failed")
assert.Equal(s.T(), res3.Priority, 3, "priority not right")
for i := 0; i < 10; i = i + 1 {
res, _, err := client.Batch.DescribeJob(context.Background(), jobid, headers)
assert.Nil(s.T(), err, "describe job Failed")
assert.Equal(s.T(), res2.Job.ConfirmationRequired, "true", "ConfirmationRequired not right")
assert.Equal(s.T(), res2.Job.Description, "test batch", "Description not right")
assert.Equal(s.T(), res2.Job.JobId, jobid, "jobid not right")
assert.Equal(s.T(), res2.Job.Priority, 1, "priority not right")
assert.Equal(s.T(), res2.Job.RoleArn, "qcs::cam::uin/"+kUin+":roleName/COSBatch_QcsRole", "priority not right")
if res.Job.Status == "Suspended" {
break
}
if i == 9 {
assert.Error(s.T(), errors.New("Job status is not Suspended or timeout"))
}
time.Sleep(time.Second * 2)
}
us_opt := &cos.BatchUpdateStatusOptions{
JobId: jobid,
RequestedJobStatus: "Ready", // 允许状态转换见 https://cloud.tencent.com/document/product/436/38604
StatusUpdateReason: "to test",
}
res4, _, err := client.Batch.UpdateJobStatus(context.Background(), us_opt, headers)
assert.Nil(s.T(), err, "list jobs failed")
assert.Equal(s.T(), res4.JobId, jobid, "jobid failed")
assert.Equal(s.T(), res4.Status, "Ready", "status failed")
assert.Equal(s.T(), res4.StatusUpdateReason, "to test", "StatusUpdateReason failed")
}
// End of api test // End of api test
// All methods that begin with "Test" are run as tests within a // All methods that begin with "Test" are run as tests within a

99
example/batch/create_job.go
View File

@ -0,0 +1,99 @@
package main
import (
"context"
"net/http"
"net/url"
"os"
"strings"
"fmt"
"github.com/tencentyun/cos-go-sdk-v5"
"github.com/tencentyun/cos-go-sdk-v5/debug"
"github.com/google/uuid"
)
func main() {
test_batch_bucket := "testcd-1259654469"
target_batch_bucket := "targetcd-1259654469"
appid := 1259654469
uin := "100010805041"
region := "ap-chengdu"
// bucket url：<Bucketname-Appid>.cos.<region>.mycloud.com
bucketurl, _ := url.Parse("https://" + test_batch_bucket + ".cos." + region + ".myqcloud.com")
// batch url：<uin>.cos-control.<region>.myqcloud.ccom
batchurl, _ := url.Parse("https://" + uin + ".cos-control." + region + ".myqcloud.com")
b := &cos.BaseURL{BucketURL: bucketurl, BatchURL: batchurl}
c := cos.NewClient(b, &http.Client{
Transport: &cos.AuthorizationTransport{
SecretID: os.Getenv("COS_SECRETID"),
SecretKey: os.Getenv("COS_SECRETKEY"),
Transport: &debug.DebugRequestTransport{
RequestHeader: true,
RequestBody: true,
ResponseHeader: true,
ResponseBody: true,
},
},
})
// 创建需要复制的文件
source_name := "test/1.txt"
sf := strings.NewReader("batch test content")
_, err := c.Object.Put(context.Background(), source_name, sf, nil)
if err != nil {
panic(err)
}
// 创建清单文件
manifest_name := "test/manifest.csv"
f := strings.NewReader(test_batch_bucket + "," + source_name)
resp, err := c.Object.Put(context.Background(), manifest_name, f, nil)
if err != nil {
panic(err)
}
etag := resp.Header.Get("ETag")
uuid_str := uuid.New().String()
opt := &cos.BatchCreateJobOptions{
ClientRequestToken: uuid_str,
ConfirmationRequired: "true",
Description: "test batch",
Manifest: &cos.BatchJobManifest{
Location: &cos.BatchJobManifestLocation{
ETag: etag,
ObjectArn: "qcs::cos:" + region + "::" + test_batch_bucket + "/" + manifest_name,
},
Spec: &cos.BatchJobManifestSpec{
Fields: []string{"Bucket", "Key"},
Format: "COSBatchOperations_CSV_V1",
},
},
Operation: &cos.BatchJobOperation{
PutObjectCopy: &cos.BatchJobOperationCopy{
TargetResource: "qcs::cos:" + region + "::" + target_batch_bucket,
},
},
Priority: 1,
Report: &cos.BatchJobReport{
Bucket: "qcs::cos:" + region + "::" + test_batch_bucket,
Enabled: "true",
Format: "Report_CSV_V1",
Prefix: "job-result",
ReportScope: "AllTasks",
},
RoleArn: "qcs::cam::uin/" + uin + ":roleName/COSBatch_QcsRole",
}
headers := &cos.BatchRequestHeaders{
XCosAppid: appid,
}
res, _, err := c.Batch.CreateJob(context.Background(), opt, headers)
if err != nil {
panic(err)
}
fmt.Println(res)
}

44
example/batch/describe_job.go
View File

@ -0,0 +1,44 @@
package main
import (
"context"
"net/http"
"net/url"
"os"
"fmt"
"github.com/tencentyun/cos-go-sdk-v5"
"github.com/tencentyun/cos-go-sdk-v5/debug"
)
func main() {
uin := "100010805041"
appid := 1259654469
jobid := "795ad997-5557-4869-9a19-b66ec087d460"
u, _ := url.Parse("https://" + uin + ".cos-control.ap-chengdu.myqcloud.com")
b := &cos.BaseURL{BatchURL: u}
c := cos.NewClient(b, &http.Client{
Transport: &cos.AuthorizationTransport{
SecretID: os.Getenv("COS_SECRETID"),
SecretKey: os.Getenv("COS_SECRETKEY"),
Transport: &debug.DebugRequestTransport{
RequestHeader: true,
RequestBody: true,
ResponseHeader: true,
ResponseBody: true,
},
},
})
headers := &cos.BatchRequestHeaders{
XCosAppid: appid,
}
res, _, err := c.Batch.DescribeJob(context.Background(), jobid, headers)
if err != nil {
panic(err)
}
if res != nil && res.Job != nil {
fmt.Printf("%+v", res.Job)
}
}

43
example/batch/list_jobs.go
View File

@ -0,0 +1,43 @@
package main
import (
"context"
"net/http"
"net/url"
"os"
"fmt"
"github.com/tencentyun/cos-go-sdk-v5"
"github.com/tencentyun/cos-go-sdk-v5/debug"
)
func main() {
uin := "100010805041"
appid := 1259654469
u, _ := url.Parse("https://" + uin + ".cos-control.ap-chengdu.myqcloud.com")
b := &cos.BaseURL{BatchURL: u}
c := cos.NewClient(b, &http.Client{
Transport: &cos.AuthorizationTransport{
SecretID: os.Getenv("COS_SECRETID"),
SecretKey: os.Getenv("COS_SECRETKEY"),
Transport: &debug.DebugRequestTransport{
RequestHeader: true,
RequestBody: true,
ResponseHeader: true,
ResponseBody: true,
},
},
})
headers := &cos.BatchRequestHeaders{
XCosAppid: appid,
}
res, _, err := c.Batch.ListJobs(context.Background(), nil, headers)
if err != nil {
panic(err)
}
if res != nil && res.Jobs != nil {
fmt.Printf("%+v", res.Jobs)
}
}

48
example/batch/update_priority.go
View File

@ -0,0 +1,48 @@
package main
import (
"context"
"net/http"
"net/url"
"os"
"fmt"
"github.com/tencentyun/cos-go-sdk-v5"
"github.com/tencentyun/cos-go-sdk-v5/debug"
)
func main() {
uin := "100010805041"
appid := 1259654469
jobid := "795ad997-5557-4869-9a19-b66ec087d460"
u, _ := url.Parse("https://" + uin + ".cos-control.ap-chengdu.myqcloud.com")
b := &cos.BaseURL{BatchURL: u}
c := cos.NewClient(b, &http.Client{
Transport: &cos.AuthorizationTransport{
SecretID: os.Getenv("COS_SECRETID"),
SecretKey: os.Getenv("COS_SECRETKEY"),
Transport: &debug.DebugRequestTransport{
RequestHeader: true,
RequestBody: true,
ResponseHeader: true,
ResponseBody: true,
},
},
})
opt := &cos.BatchUpdatePriorityOptions{
JobId: jobid,
Priority: 3,
}
headers := &cos.BatchRequestHeaders{
XCosAppid: appid,
}
res, _, err := c.Batch.UpdateJobPriority(context.Background(), opt, headers)
if err != nil {
panic(err)
}
if res != nil {
fmt.Printf("%+v", res)
}
}

49
example/batch/update_status.go
View File

@ -0,0 +1,49 @@
package main
import (
"context"
"net/http"
"net/url"
"os"
"fmt"
"github.com/tencentyun/cos-go-sdk-v5"
"github.com/tencentyun/cos-go-sdk-v5/debug"
)
func main() {
uin := "100010805041"
appid := 1259654469
jobid := "289b0ea1-5ac5-453d-8a61-7f452dd4a209"
u, _ := url.Parse("https://" + uin + ".cos-control.ap-chengdu.myqcloud.com")
b := &cos.BaseURL{BatchURL: u}
c := cos.NewClient(b, &http.Client{
Transport: &cos.AuthorizationTransport{
SecretID: os.Getenv("COS_SECRETID"),
SecretKey: os.Getenv("COS_SECRETKEY"),
Transport: &debug.DebugRequestTransport{
RequestHeader: true,
RequestBody: true,
ResponseHeader: true,
ResponseBody: true,
},
},
})
opt := &cos.BatchUpdateStatusOptions{
JobId: jobid,
RequestedJobStatus: "Ready", // 允许状态转换见 https://cloud.tencent.com/document/product/436/38604
StatusUpdateReason: "to test",
}
headers := &cos.BatchRequestHeaders{
XCosAppid: appid,
}
res, _, err := c.Batch.UpdateJobStatus(context.Background(), opt, headers)
if err != nil {
panic(err)
}
if res != nil {
fmt.Printf("%+v", res)
}
}
Write Preview
Loading…
Cancel
Save