agin719
4 years ago
committed by
GitHub
GitHub
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 1231 additions and 15 deletions
-
177ci.go
-
10cos.go
-
2cos_test.go
-
63example/object/ci_image_process.go
-
56example/object/ci_image_recognition.go
-
71example/object/ci_video_auditing_job.go
-
98example/object/list_uploads.go
-
66example/object/select.go
-
66example/object/select_csv.go
-
17helper.go
-
125object.go
-
51object_part.go
-
444object_select.go
@ -0,0 +1,63 @@ |
|||||
|
package main |
||||
|
|
||||
|
import ( |
||||
|
"context" |
||||
|
"fmt" |
||||
|
"net/http" |
||||
|
"net/url" |
||||
|
"os" |
||||
|
|
||||
|
"github.com/tencentyun/cos-go-sdk-v5" |
||||
|
"github.com/tencentyun/cos-go-sdk-v5/debug" |
||||
|
) |
||||
|
|
||||
|
func log_status(err error) { |
||||
|
if err == nil { |
||||
|
return |
||||
|
} |
||||
|
if cos.IsNotFoundError(err) { |
||||
|
// WARN
|
||||
|
fmt.Println("WARN: Resource is not existed") |
||||
|
} else if e, ok := cos.IsCOSError(err); ok { |
||||
|
fmt.Printf("ERROR: Code: %v\n", e.Code) |
||||
|
fmt.Printf("ERROR: Message: %v\n", e.Message) |
||||
|
fmt.Printf("ERROR: Resource: %v\n", e.Resource) |
||||
|
fmt.Printf("ERROR: RequestId: %v\n", e.RequestID) |
||||
|
// ERROR
|
||||
|
} else { |
||||
|
fmt.Printf("ERROR: %v\n", err) |
||||
|
// ERROR
|
||||
|
} |
||||
|
} |
||||
|
|
||||
|
func main() { |
||||
|
u, _ := url.Parse("https://test-1259654469.cos.ap-guangzhou.myqcloud.com") |
||||
|
b := &cos.BaseURL{BucketURL: u} |
||||
|
c := cos.NewClient(b, &http.Client{ |
||||
|
Transport: &cos.AuthorizationTransport{ |
||||
|
SecretID: os.Getenv("COS_SECRETID"), |
||||
|
SecretKey: os.Getenv("COS_SECRETKEY"), |
||||
|
Transport: &debug.DebugRequestTransport{ |
||||
|
RequestHeader: true, |
||||
|
// Notice when put a large file and set need the request body, might happend out of memory error.
|
||||
|
RequestBody: false, |
||||
|
ResponseHeader: true, |
||||
|
ResponseBody: true, |
||||
|
}, |
||||
|
}, |
||||
|
}) |
||||
|
|
||||
|
opt := &cos.ImageProcessOptions{ |
||||
|
IsPicInfo: 1, |
||||
|
Rules: []cos.PicOperationsRules{ |
||||
|
{ |
||||
|
FileId: "format.jpg", |
||||
|
Rule: "imageView2/format/png", |
||||
|
}, |
||||
|
}, |
||||
|
} |
||||
|
name := "test.jpg" |
||||
|
res, _, err := c.CI.ImageProcess(context.Background(), name, opt) |
||||
|
log_status(err) |
||||
|
fmt.Printf("%+v\n", res) |
||||
|
} |
||||
@ -0,0 +1,56 @@ |
|||||
|
package main |
||||
|
|
||||
|
import ( |
||||
|
"context" |
||||
|
"fmt" |
||||
|
"net/http" |
||||
|
"net/url" |
||||
|
"os" |
||||
|
|
||||
|
"github.com/tencentyun/cos-go-sdk-v5" |
||||
|
"github.com/tencentyun/cos-go-sdk-v5/debug" |
||||
|
) |
||||
|
|
||||
|
func log_status(err error) { |
||||
|
if err == nil { |
||||
|
return |
||||
|
} |
||||
|
if cos.IsNotFoundError(err) { |
||||
|
// WARN
|
||||
|
fmt.Println("WARN: Resource is not existed") |
||||
|
} else if e, ok := cos.IsCOSError(err); ok { |
||||
|
fmt.Printf("ERROR: Code: %v\n", e.Code) |
||||
|
fmt.Printf("ERROR: Message: %v\n", e.Message) |
||||
|
fmt.Printf("ERROR: Resource: %v\n", e.Resource) |
||||
|
fmt.Printf("ERROR: RequestId: %v\n", e.RequestID) |
||||
|
// ERROR
|
||||
|
} else { |
||||
|
fmt.Printf("ERROR: %v\n", err) |
||||
|
// ERROR
|
||||
|
} |
||||
|
} |
||||
|
|
||||
|
func main() { |
||||
|
u, _ := url.Parse("https://test-1259654469.cos.ap-guangzhou.myqcloud.com") |
||||
|
b := &cos.BaseURL{BucketURL: u} |
||||
|
c := cos.NewClient(b, &http.Client{ |
||||
|
Transport: &cos.AuthorizationTransport{ |
||||
|
SecretID: os.Getenv("COS_SECRETID"), |
||||
|
SecretKey: os.Getenv("COS_SECRETKEY"), |
||||
|
Transport: &debug.DebugRequestTransport{ |
||||
|
RequestHeader: true, |
||||
|
RequestBody: true, |
||||
|
ResponseHeader: true, |
||||
|
ResponseBody: true, |
||||
|
}, |
||||
|
}, |
||||
|
}) |
||||
|
opt := &cos.ImageRecognitionOptions{ |
||||
|
DetectType: "porn,terrorist,politics", |
||||
|
} |
||||
|
|
||||
|
name := "test.jpg" |
||||
|
res, _, err := c.CI.ImageRecognition(context.Background(), name, opt) |
||||
|
log_status(err) |
||||
|
fmt.Printf("%+v\n", res) |
||||
|
} |
||||
@ -0,0 +1,71 @@ |
|||||
|
package main |
||||
|
|
||||
|
import ( |
||||
|
"context" |
||||
|
"fmt" |
||||
|
"net/http" |
||||
|
"net/url" |
||||
|
"os" |
||||
|
"time" |
||||
|
|
||||
|
"github.com/tencentyun/cos-go-sdk-v5" |
||||
|
"github.com/tencentyun/cos-go-sdk-v5/debug" |
||||
|
) |
||||
|
|
||||
|
func log_status(err error) { |
||||
|
if err == nil { |
||||
|
return |
||||
|
} |
||||
|
if cos.IsNotFoundError(err) { |
||||
|
// WARN
|
||||
|
fmt.Println("WARN: Resource is not existed") |
||||
|
} else if e, ok := cos.IsCOSError(err); ok { |
||||
|
fmt.Printf("ERROR: Code: %v\n", e.Code) |
||||
|
fmt.Printf("ERROR: Message: %v\n", e.Message) |
||||
|
fmt.Printf("ERROR: Resource: %v\n", e.Resource) |
||||
|
fmt.Printf("ERROR: RequestId: %v\n", e.RequestID) |
||||
|
// ERROR
|
||||
|
} else { |
||||
|
fmt.Printf("ERROR: %v\n", err) |
||||
|
// ERROR
|
||||
|
} |
||||
|
} |
||||
|
|
||||
|
func main() { |
||||
|
bu, _ := url.Parse("https://test-1259654469.cos.ap-guangzhou.myqcloud.com") |
||||
|
cu, _ := url.Parse("https://test-1259654469.ci.ap-guangzhou.myqcloud.com") |
||||
|
b := &cos.BaseURL{BucketURL: bu, CIURL: cu} |
||||
|
c := cos.NewClient(b, &http.Client{ |
||||
|
Transport: &cos.AuthorizationTransport{ |
||||
|
SecretID: os.Getenv("COS_SECRETID"), |
||||
|
SecretKey: os.Getenv("COS_SECRETKEY"), |
||||
|
Transport: &debug.DebugRequestTransport{ |
||||
|
RequestHeader: true, |
||||
|
RequestBody: true, |
||||
|
ResponseHeader: true, |
||||
|
ResponseBody: true, |
||||
|
}, |
||||
|
}, |
||||
|
}) |
||||
|
opt := &cos.PutVideoAuditingJobOptions{ |
||||
|
InputObject: "demo.mp4", |
||||
|
Conf: &cos.VideoAuditingJobConf{ |
||||
|
DetectType: "Porn,Terrorism,Politics,Ads", |
||||
|
Snapshot: &cos.PutVideoAuditingJobSnapshot{ |
||||
|
Mode: "Interval", |
||||
|
Start: 0.5, |
||||
|
TimeInterval: 50.5, |
||||
|
Count: 100, |
||||
|
}, |
||||
|
}, |
||||
|
} |
||||
|
|
||||
|
res, _, err := c.CI.PutVideoAuditingJob(context.Background(), opt) |
||||
|
log_status(err) |
||||
|
fmt.Printf("%+v\n", res) |
||||
|
|
||||
|
time.Sleep(3 * time.Second) |
||||
|
res2, _, err := c.CI.GetVideoAuditingJob(context.Background(), res.JobsDetail.JobId) |
||||
|
log_status(err) |
||||
|
fmt.Printf("%+v\n", res2) |
||||
|
} |
||||
@ -0,0 +1,98 @@ |
|||||
|
package main |
||||
|
|
||||
|
import ( |
||||
|
"context" |
||||
|
"fmt" |
||||
|
"math/rand" |
||||
|
"net/url" |
||||
|
"os" |
||||
|
"strings" |
||||
|
|
||||
|
"net/http" |
||||
|
|
||||
|
"github.com/tencentyun/cos-go-sdk-v5" |
||||
|
"github.com/tencentyun/cos-go-sdk-v5/debug" |
||||
|
) |
||||
|
|
||||
|
func log_status(err error) { |
||||
|
if err == nil { |
||||
|
return |
||||
|
} |
||||
|
if cos.IsNotFoundError(err) { |
||||
|
// WARN
|
||||
|
fmt.Println("WARN: Resource is not existed") |
||||
|
} else if e, ok := cos.IsCOSError(err); ok { |
||||
|
fmt.Printf("ERROR: Code: %v\n", e.Code) |
||||
|
fmt.Printf("ERROR: Message: %v\n", e.Message) |
||||
|
fmt.Printf("ERROR: Resource: %v\n", e.Resource) |
||||
|
fmt.Printf("ERROR: RequestId: %v\n", e.RequestID) |
||||
|
// ERROR
|
||||
|
} else { |
||||
|
fmt.Printf("ERROR: %v\n", err) |
||||
|
// ERROR
|
||||
|
} |
||||
|
} |
||||
|
|
||||
|
func initUpload(c *cos.Client, name string) *cos.InitiateMultipartUploadResult { |
||||
|
v, _, err := c.Object.InitiateMultipartUpload(context.Background(), name, nil) |
||||
|
log_status(err) |
||||
|
fmt.Printf("%#v\n", v) |
||||
|
return v |
||||
|
} |
||||
|
|
||||
|
func uploadPart(c *cos.Client, name string, uploadID string, blockSize, n int) string { |
||||
|
|
||||
|
b := make([]byte, blockSize) |
||||
|
if _, err := rand.Read(b); err != nil { |
||||
|
log_status(err) |
||||
|
} |
||||
|
s := fmt.Sprintf("%X", b) |
||||
|
f := strings.NewReader(s) |
||||
|
|
||||
|
resp, err := c.Object.UploadPart( |
||||
|
context.Background(), name, uploadID, n, f, nil, |
||||
|
) |
||||
|
log_status(err) |
||||
|
fmt.Printf("%s\n", resp.Status) |
||||
|
return resp.Header.Get("Etag") |
||||
|
} |
||||
|
|
||||
|
func main() { |
||||
|
u, _ := url.Parse("http://test-1259654469.cos.ap-guangzhou.myqcloud.com") |
||||
|
b := &cos.BaseURL{BucketURL: u} |
||||
|
c := cos.NewClient(b, &http.Client{ |
||||
|
Transport: &cos.AuthorizationTransport{ |
||||
|
SecretID: os.Getenv("COS_SECRETID"), |
||||
|
SecretKey: os.Getenv("COS_SECRETKEY"), |
||||
|
Transport: &debug.DebugRequestTransport{ |
||||
|
RequestHeader: true, |
||||
|
RequestBody: false, |
||||
|
ResponseHeader: true, |
||||
|
ResponseBody: true, |
||||
|
}, |
||||
|
}, |
||||
|
}) |
||||
|
|
||||
|
name := "test/test_list_parts.go" |
||||
|
up := initUpload(c, name) |
||||
|
uploadID := up.UploadID |
||||
|
blockSize := 1024 * 1024 * 3 |
||||
|
|
||||
|
for i := 1; i < 5; i++ { |
||||
|
uploadPart(c, name, uploadID, blockSize, i) |
||||
|
} |
||||
|
opt := &cos.ObjectListUploadsOptions{ |
||||
|
Prefix: cos.EncodeURIComponent("test/test_list_parts"), |
||||
|
MaxUploads: 100, |
||||
|
} |
||||
|
v, _, err := c.Object.ListUploads(context.Background(), opt) |
||||
|
if err != nil { |
||||
|
log_status(err) |
||||
|
return |
||||
|
} |
||||
|
fmt.Printf("%+v\n", v) |
||||
|
for _, p := range v.Upload { |
||||
|
fmt.Printf("%+v\n", p) |
||||
|
fmt.Printf("%v, %v, %v\n", p.Key, p.UploadID, p.Initiated) |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,66 @@ |
|||||
|
package main |
||||
|
|
||||
|
import ( |
||||
|
"context" |
||||
|
"fmt" |
||||
|
"net/url" |
||||
|
"os" |
||||
|
|
||||
|
"io/ioutil" |
||||
|
"net/http" |
||||
|
|
||||
|
"github.com/tencentyun/cos-go-sdk-v5" |
||||
|
"github.com/tencentyun/cos-go-sdk-v5/debug" |
||||
|
) |
||||
|
|
||||
|
func main() { |
||||
|
u, _ := url.Parse("https://test-1259654469.cos.ap-guangzhou.myqcloud.com") |
||||
|
b := &cos.BaseURL{BucketURL: u} |
||||
|
c := cos.NewClient(b, &http.Client{ |
||||
|
Transport: &cos.AuthorizationTransport{ |
||||
|
SecretID: os.Getenv("COS_SECRETID"), |
||||
|
SecretKey: os.Getenv("COS_SECRETKEY"), |
||||
|
Transport: &debug.DebugRequestTransport{ |
||||
|
RequestHeader: true, |
||||
|
// Notice when put a large file and set need the request body, might happend out of memory error.
|
||||
|
RequestBody: false, |
||||
|
ResponseHeader: true, |
||||
|
ResponseBody: false, |
||||
|
}, |
||||
|
}, |
||||
|
}) |
||||
|
|
||||
|
opt := &cos.ObjectSelectOptions{ |
||||
|
Expression: "Select * from COSObject", |
||||
|
ExpressionType: "SQL", |
||||
|
InputSerialization: &cos.SelectInputSerialization{ |
||||
|
JSON: &cos.JSONInputSerialization{ |
||||
|
Type: "DOCUMENT", |
||||
|
}, |
||||
|
}, |
||||
|
OutputSerialization: &cos.SelectOutputSerialization{ |
||||
|
JSON: &cos.JSONOutputSerialization{ |
||||
|
RecordDelimiter: "\n", |
||||
|
}, |
||||
|
}, |
||||
|
RequestProgress: "TRUE", |
||||
|
} |
||||
|
res, err := c.Object.Select(context.Background(), "test.json", opt) |
||||
|
if err != nil { |
||||
|
panic(err) |
||||
|
} |
||||
|
defer res.Close() |
||||
|
data, err := ioutil.ReadAll(res) |
||||
|
if err != nil { |
||||
|
panic(err) |
||||
|
} |
||||
|
fmt.Printf("data: %v\n", string(data)) |
||||
|
resp, _ := res.(*cos.ObjectSelectResponse) |
||||
|
fmt.Printf("data: %+v\n", resp.Frame) |
||||
|
|
||||
|
// Select to File
|
||||
|
_, err = c.Object.SelectToFile(context.Background(), "test.json", "./test.json", opt) |
||||
|
if err != nil { |
||||
|
panic(err) |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,66 @@ |
|||||
|
package main |
||||
|
|
||||
|
import ( |
||||
|
"context" |
||||
|
"fmt" |
||||
|
"net/url" |
||||
|
"os" |
||||
|
|
||||
|
"io/ioutil" |
||||
|
"net/http" |
||||
|
|
||||
|
"github.com/tencentyun/cos-go-sdk-v5" |
||||
|
"github.com/tencentyun/cos-go-sdk-v5/debug" |
||||
|
) |
||||
|
|
||||
|
func main() { |
||||
|
u, _ := url.Parse("https://test-1259654469.cos.ap-guangzhou.myqcloud.com") |
||||
|
b := &cos.BaseURL{BucketURL: u} |
||||
|
c := cos.NewClient(b, &http.Client{ |
||||
|
Transport: &cos.AuthorizationTransport{ |
||||
|
SecretID: os.Getenv("COS_SECRETID"), |
||||
|
SecretKey: os.Getenv("COS_SECRETKEY"), |
||||
|
Transport: &debug.DebugRequestTransport{ |
||||
|
RequestHeader: true, |
||||
|
// Notice when put a large file and set need the request body, might happend out of memory error.
|
||||
|
RequestBody: false, |
||||
|
ResponseHeader: true, |
||||
|
ResponseBody: false, |
||||
|
}, |
||||
|
}, |
||||
|
}) |
||||
|
|
||||
|
opt := &cos.ObjectSelectOptions{ |
||||
|
Expression: "Select * from COSObject", |
||||
|
ExpressionType: "SQL", |
||||
|
InputSerialization: &cos.SelectInputSerialization{ |
||||
|
CSV: &cos.CSVInputSerialization{ |
||||
|
FileHeaderInfo: "IGNORE", |
||||
|
}, |
||||
|
}, |
||||
|
OutputSerialization: &cos.SelectOutputSerialization{ |
||||
|
CSV: &cos.CSVOutputSerialization{ |
||||
|
RecordDelimiter: "\n", |
||||
|
}, |
||||
|
}, |
||||
|
RequestProgress: "TRUE", |
||||
|
} |
||||
|
res, err := c.Object.Select(context.Background(), "test.csv", opt) |
||||
|
if err != nil { |
||||
|
panic(err) |
||||
|
} |
||||
|
defer res.Close() |
||||
|
data, err := ioutil.ReadAll(res) |
||||
|
if err != nil { |
||||
|
panic(err) |
||||
|
} |
||||
|
fmt.Printf("data: %v\n", string(data)) |
||||
|
resp, _ := res.(*cos.ObjectSelectResponse) |
||||
|
fmt.Printf("data: %+v\n", resp.Frame) |
||||
|
|
||||
|
// Select To File
|
||||
|
_, err = c.Object.SelectToFile(context.Background(), "test.csv", "./test.csv", opt) |
||||
|
if err != nil { |
||||
|
panic(err) |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,444 @@ |
|||||
|
package cos |
||||
|
|
||||
|
import ( |
||||
|
"bytes" |
||||
|
"context" |
||||
|
"encoding/binary" |
||||
|
"encoding/xml" |
||||
|
"fmt" |
||||
|
"hash/crc32" |
||||
|
"io" |
||||
|
"io/ioutil" |
||||
|
"net/http" |
||||
|
"os" |
||||
|
"time" |
||||
|
) |
||||
|
|
||||
|
type JSONInputSerialization struct { |
||||
|
Type string `xml:"Type"` |
||||
|
} |
||||
|
|
||||
|
type CSVInputSerialization struct { |
||||
|
RecordDelimiter string `xml:"RecordDelimiter,omitempty"` |
||||
|
FieldDelimiter string `xml:"FieldDelimiter,omitempty"` |
||||
|
QuoteCharacter string `xml:"QuoteCharacter,omitempty"` |
||||
|
QuoteEscapeCharacter string `xml:"QuoteEscapeCharacter,omitempty"` |
||||
|
AllowQuotedRecordDelimiter string `xml:"AllowQuotedRecordDelimiter,omitempty"` |
||||
|
FileHeaderInfo string `xml:"FileHeaderInfo,omitempty"` |
||||
|
Comments string `xml:"Comments,omitempty"` |
||||
|
} |
||||
|
|
||||
|
type SelectInputSerialization struct { |
||||
|
CompressionType string `xml:"CompressionType,omitempty"` |
||||
|
CSV *CSVInputSerialization `xml:"CSV,omitempty"` |
||||
|
JSON *JSONInputSerialization `xml:"JSON,omitempty"` |
||||
|
} |
||||
|
|
||||
|
type JSONOutputSerialization struct { |
||||
|
RecordDelimiter string `xml:"RecordDelimiter,omitempty"` |
||||
|
} |
||||
|
|
||||
|
type CSVOutputSerialization struct { |
||||
|
QuoteFileds string `xml:"QuoteFileds,omitempty"` |
||||
|
RecordDelimiter string `xml:"RecordDelimiter,omitempty"` |
||||
|
FieldDelimiter string `xml:"FieldDelimiter,omitempty"` |
||||
|
QuoteCharacter string `xml:"QuoteCharacter,omitempty"` |
||||
|
QuoteEscapeCharacter string `xml:"QuoteEscapeCharacter,omitempty"` |
||||
|
} |
||||
|
|
||||
|
type SelectOutputSerialization struct { |
||||
|
CSV *CSVOutputSerialization `xml:"CSV,omitempty"` |
||||
|
JSON *JSONOutputSerialization `xml:"JSON,omitempty"` |
||||
|
} |
||||
|
|
||||
|
type ObjectSelectOptions struct { |
||||
|
XMLName xml.Name `xml:"SelectRequest"` |
||||
|
Expression string `xml:"Expression"` |
||||
|
ExpressionType string `xml:"ExpressionType"` |
||||
|
InputSerialization *SelectInputSerialization `xml:"InputSerialization"` |
||||
|
OutputSerialization *SelectOutputSerialization `xml:"OutputSerialization"` |
||||
|
RequestProgress string `xml:"RequestProgress>Enabled,omitempty"` |
||||
|
} |
||||
|
|
||||
|
func (s *ObjectService) Select(ctx context.Context, name string, opt *ObjectSelectOptions) (io.ReadCloser, error) { |
||||
|
u := fmt.Sprintf("/%s?select&select-type=2", encodeURIComponent(name)) |
||||
|
sendOpt := sendOptions{ |
||||
|
baseURL: s.client.BaseURL.BucketURL, |
||||
|
uri: u, |
||||
|
method: http.MethodPost, |
||||
|
body: opt, |
||||
|
disableCloseBody: true, |
||||
|
} |
||||
|
resp, err := s.client.send(ctx, &sendOpt) |
||||
|
if err != nil { |
||||
|
return nil, err |
||||
|
} |
||||
|
result := &ObjectSelectResponse{ |
||||
|
Headers: resp.Header, |
||||
|
Body: resp.Body, |
||||
|
StatusCode: resp.StatusCode, |
||||
|
Frame: &ObjectSelectResult{ |
||||
|
NextFrame: true, |
||||
|
Payload: []byte{}, |
||||
|
}, |
||||
|
Finish: false, |
||||
|
} |
||||
|
|
||||
|
return result, nil |
||||
|
} |
||||
|
|
||||
|
func (s *ObjectService) SelectToFile(ctx context.Context, name, file string, opt *ObjectSelectOptions) (*ObjectSelectResponse, error) { |
||||
|
resp, err := s.Select(ctx, name, opt) |
||||
|
if err != nil { |
||||
|
return nil, err |
||||
|
} |
||||
|
res, _ := resp.(*ObjectSelectResponse) |
||||
|
defer func() { |
||||
|
io.Copy(ioutil.Discard, resp) |
||||
|
resp.Close() |
||||
|
}() |
||||
|
|
||||
|
fd, err := os.OpenFile(file, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, os.FileMode(0664)) |
||||
|
if err != nil { |
||||
|
return res, err |
||||
|
} |
||||
|
|
||||
|
_, err = io.Copy(fd, resp) |
||||
|
fd.Close() |
||||
|
res.Finish = true |
||||
|
return res, err |
||||
|
} |
||||
|
|
||||
|
const ( |
||||
|
kReadTimeout = 3 |
||||
|
kMessageType = ":message-type" |
||||
|
kEventType = ":event-type" |
||||
|
kContentType = ":content-type" |
||||
|
|
||||
|
kRecordsFrameType = iota |
||||
|
kContinuationFrameType |
||||
|
kProgressFrameType |
||||
|
kStatsFrameType |
||||
|
kEndFrameType |
||||
|
kErrorFrameType |
||||
|
) |
||||
|
|
||||
|
type ProgressFrame struct { |
||||
|
XMLName xml.Name `xml:"Progress"` |
||||
|
BytesScanned int `xml:"BytesScanned"` |
||||
|
BytesProcessed int `xml:"BytesProcessed"` |
||||
|
BytesReturned int `xml:"BytesReturned"` |
||||
|
} |
||||
|
|
||||
|
type StatsFrame struct { |
||||
|
XMLName xml.Name `xml:"Stats"` |
||||
|
BytesScanned int `xml:"BytesScanned"` |
||||
|
BytesProcessed int `xml:"BytesProcessed"` |
||||
|
BytesReturned int `xml:"BytesReturned"` |
||||
|
} |
||||
|
|
||||
|
type DataFrame struct { |
||||
|
ContentType string |
||||
|
ConsumedBytesLength int32 |
||||
|
LeftBytesLength int32 |
||||
|
} |
||||
|
|
||||
|
type ErrorFrame struct { |
||||
|
Code string |
||||
|
Message string |
||||
|
} |
||||
|
|
||||
|
func (e *ErrorFrame) Error() string { |
||||
|
return fmt.Sprintf("Error Code: %s, Error Message: %s", e.Code, e.Message) |
||||
|
} |
||||
|
|
||||
|
type ObjectSelectResult struct { |
||||
|
TotalFrameLength int32 |
||||
|
TotalHeaderLength int32 |
||||
|
NextFrame bool |
||||
|
FrameType int |
||||
|
Payload []byte |
||||
|
DataFrame DataFrame |
||||
|
ProgressFrame ProgressFrame |
||||
|
StatsFrame StatsFrame |
||||
|
ErrorFrame *ErrorFrame |
||||
|
} |
||||
|
|
||||
|
type ObjectSelectResponse struct { |
||||
|
StatusCode int |
||||
|
Headers http.Header |
||||
|
Body io.ReadCloser |
||||
|
Frame *ObjectSelectResult |
||||
|
Finish bool |
||||
|
} |
||||
|
|
||||
|
func (osr *ObjectSelectResponse) Read(p []byte) (n int, err error) { |
||||
|
n, err = osr.readFrames(p) |
||||
|
return |
||||
|
} |
||||
|
func (osr *ObjectSelectResponse) Close() error { |
||||
|
return osr.Body.Close() |
||||
|
} |
||||
|
|
||||
|
func (osr *ObjectSelectResponse) readFrames(p []byte) (int, error) { |
||||
|
if osr.Finish { |
||||
|
return 0, io.EOF |
||||
|
} |
||||
|
if osr.Frame.ErrorFrame != nil { |
||||
|
return 0, osr.Frame.ErrorFrame |
||||
|
} |
||||
|
|
||||
|
var err error |
||||
|
var nlen int |
||||
|
dlen := len(p) |
||||
|
|
||||
|
for nlen < dlen { |
||||
|
if osr.Frame.NextFrame == true { |
||||
|
osr.Frame.NextFrame = false |
||||
|
err := osr.analysisPrelude() |
||||
|
if err != nil { |
||||
|
return nlen, err |
||||
|
} |
||||
|
err = osr.analysisHeader() |
||||
|
if err != nil { |
||||
|
return nlen, err |
||||
|
} |
||||
|
} |
||||
|
switch osr.Frame.FrameType { |
||||
|
case kRecordsFrameType: |
||||
|
n, err := osr.analysisRecords(p[nlen:]) |
||||
|
if err != nil { |
||||
|
return nlen, err |
||||
|
} |
||||
|
nlen += n |
||||
|
case kContinuationFrameType: |
||||
|
err = osr.payloadChecksum("ContinuationFrame") |
||||
|
if err != nil { |
||||
|
return nlen, err |
||||
|
} |
||||
|
case kProgressFrameType: |
||||
|
err := osr.analysisXml(&osr.Frame.ProgressFrame) |
||||
|
if err != nil { |
||||
|
return nlen, err |
||||
|
} |
||||
|
case kStatsFrameType: |
||||
|
err := osr.analysisXml(&osr.Frame.StatsFrame) |
||||
|
if err != nil { |
||||
|
return nlen, err |
||||
|
} |
||||
|
case kEndFrameType: |
||||
|
err = osr.payloadChecksum("EndFrame") |
||||
|
if err != nil { |
||||
|
return nlen, err |
||||
|
} |
||||
|
osr.Finish = true |
||||
|
return nlen, io.EOF |
||||
|
case kErrorFrameType: |
||||
|
return nlen, osr.Frame.ErrorFrame |
||||
|
} |
||||
|
} |
||||
|
return nlen, err |
||||
|
} |
||||
|
|
||||
|
func (osr *ObjectSelectResponse) analysisPrelude() error { |
||||
|
frame := make([]byte, 12) |
||||
|
_, err := osr.fixedLengthRead(frame, kReadTimeout) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
var preludeCRC uint32 |
||||
|
bytesToInt(frame[0:4], &osr.Frame.TotalFrameLength) |
||||
|
bytesToInt(frame[4:8], &osr.Frame.TotalHeaderLength) |
||||
|
bytesToInt(frame[8:12], &preludeCRC) |
||||
|
osr.Frame.Payload = append(osr.Frame.Payload, frame...) |
||||
|
|
||||
|
return checksum(frame[0:8], preludeCRC, "Prelude") |
||||
|
} |
||||
|
|
||||
|
func (osr *ObjectSelectResponse) analysisHeader() error { |
||||
|
var nlen int32 |
||||
|
headers := make(map[string]string) |
||||
|
for nlen < osr.Frame.TotalHeaderLength { |
||||
|
var headerNameLen int8 |
||||
|
var headerValueLen int16 |
||||
|
bHeaderNameLen := make([]byte, 1) |
||||
|
_, err := osr.fixedLengthRead(bHeaderNameLen, kReadTimeout) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
nlen += 1 |
||||
|
bytesToInt(bHeaderNameLen, &headerNameLen) |
||||
|
osr.Frame.Payload = append(osr.Frame.Payload, bHeaderNameLen...) |
||||
|
|
||||
|
bHeaderName := make([]byte, headerNameLen) |
||||
|
_, err = osr.fixedLengthRead(bHeaderName, kReadTimeout) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
nlen += int32(headerNameLen) |
||||
|
headerName := string(bHeaderName) |
||||
|
osr.Frame.Payload = append(osr.Frame.Payload, bHeaderName...) |
||||
|
|
||||
|
bValueTypeLen := make([]byte, 3) |
||||
|
_, err = osr.fixedLengthRead(bValueTypeLen, kReadTimeout) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
nlen += 3 |
||||
|
bytesToInt(bValueTypeLen[1:], &headerValueLen) |
||||
|
osr.Frame.Payload = append(osr.Frame.Payload, bValueTypeLen...) |
||||
|
|
||||
|
bHeaderValue := make([]byte, headerValueLen) |
||||
|
_, err = osr.fixedLengthRead(bHeaderValue, kReadTimeout) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
nlen += int32(headerValueLen) |
||||
|
headers[headerName] = string(bHeaderValue) |
||||
|
osr.Frame.Payload = append(osr.Frame.Payload, bHeaderValue...) |
||||
|
} |
||||
|
htype, ok := headers[kMessageType] |
||||
|
if !ok { |
||||
|
return fmt.Errorf("header parse failed, no message-type, headers: %+v\n", headers) |
||||
|
} |
||||
|
switch { |
||||
|
case htype == "error": |
||||
|
osr.Frame.FrameType = kErrorFrameType |
||||
|
osr.Frame.ErrorFrame = &ErrorFrame{} |
||||
|
osr.Frame.ErrorFrame.Code, _ = headers[":error-code"] |
||||
|
osr.Frame.ErrorFrame.Message, _ = headers[":error-message"] |
||||
|
case htype == "event": |
||||
|
hevent, ok := headers[kEventType] |
||||
|
if !ok { |
||||
|
return fmt.Errorf("header parse failed, no event-type, headers: %+v\n", headers) |
||||
|
} |
||||
|
switch { |
||||
|
case hevent == "Records": |
||||
|
hContentType, ok := headers[kContentType] |
||||
|
if ok { |
||||
|
osr.Frame.DataFrame.ContentType = hContentType |
||||
|
} |
||||
|
osr.Frame.FrameType = kRecordsFrameType |
||||
|
case hevent == "Cont": |
||||
|
osr.Frame.FrameType = kContinuationFrameType |
||||
|
case hevent == "Progress": |
||||
|
osr.Frame.FrameType = kProgressFrameType |
||||
|
case hevent == "Stats": |
||||
|
osr.Frame.FrameType = kStatsFrameType |
||||
|
case hevent == "End": |
||||
|
osr.Frame.FrameType = kEndFrameType |
||||
|
default: |
||||
|
return fmt.Errorf("header parse failed, invalid event-type, headers: %+v\n", headers) |
||||
|
} |
||||
|
default: |
||||
|
return fmt.Errorf("header parse failed, invalid message-type: headers: %+v\n", headers) |
||||
|
} |
||||
|
return nil |
||||
|
} |
||||
|
|
||||
|
func (osr *ObjectSelectResponse) analysisRecords(data []byte) (int, error) { |
||||
|
var needReadLength int32 |
||||
|
dlen := int32(len(data)) |
||||
|
restLen := osr.Frame.TotalFrameLength - 16 - osr.Frame.TotalHeaderLength - osr.Frame.DataFrame.ConsumedBytesLength |
||||
|
if dlen <= restLen { |
||||
|
needReadLength = dlen |
||||
|
} else { |
||||
|
needReadLength = restLen |
||||
|
} |
||||
|
n, err := osr.fixedLengthRead(data[:needReadLength], kReadTimeout) |
||||
|
if err != nil { |
||||
|
return n, fmt.Errorf("read data frame error: %s", err.Error()) |
||||
|
} |
||||
|
osr.Frame.DataFrame.ConsumedBytesLength += int32(n) |
||||
|
osr.Frame.Payload = append(osr.Frame.Payload, data[:needReadLength]...) |
||||
|
// 读完了一帧数据并填充到data中了
|
||||
|
if osr.Frame.DataFrame.ConsumedBytesLength == osr.Frame.TotalFrameLength-16-osr.Frame.TotalHeaderLength { |
||||
|
osr.Frame.DataFrame.ConsumedBytesLength = 0 |
||||
|
err = osr.payloadChecksum("RecordFrame") |
||||
|
} |
||||
|
return n, err |
||||
|
} |
||||
|
|
||||
|
func (osr *ObjectSelectResponse) analysisXml(frame interface{}) error { |
||||
|
payloadLength := osr.Frame.TotalFrameLength - 16 - osr.Frame.TotalHeaderLength |
||||
|
bFrame := make([]byte, payloadLength) |
||||
|
_, err := osr.fixedLengthRead(bFrame, kReadTimeout) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
err = xml.Unmarshal(bFrame, frame) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
osr.Frame.Payload = append(osr.Frame.Payload, bFrame...) |
||||
|
return osr.payloadChecksum("XmlFrame") |
||||
|
} |
||||
|
|
||||
|
// 调用payloadChecksum时，表示该帧已读完，开始读取下一帧内容
|
||||
|
func (osr *ObjectSelectResponse) payloadChecksum(ftype string) error { |
||||
|
bcrc := make([]byte, 4) |
||||
|
_, err := osr.fixedLengthRead(bcrc, kReadTimeout) |
||||
|
if err != nil { |
||||
|
return err |
||||
|
} |
||||
|
var res uint32 |
||||
|
bytesToInt(bcrc, &res) |
||||
|
err = checksum(osr.Frame.Payload, res, ftype) |
||||
|
|
||||
|
osr.Frame.NextFrame = true |
||||
|
osr.Frame.Payload = []byte{} |
||||
|
|
||||
|
return err |
||||
|
} |
||||
|
|
||||
|
type chanReadIO struct { |
||||
|
readLen int |
||||
|
err error |
||||
|
} |
||||
|
|
||||
|
func (osr *ObjectSelectResponse) fixedLengthRead(p []byte, read_timeout int64) (int, error) { |
||||
|
timeout := time.Duration(read_timeout) |
||||
|
r := osr.Body |
||||
|
ch := make(chan chanReadIO, 1) |
||||
|
defer close(ch) |
||||
|
go func(p []byte) { |
||||
|
var needLen int |
||||
|
readChan := chanReadIO{} |
||||
|
needLen = len(p) |
||||
|
for { |
||||
|
n, err := r.Read(p[readChan.readLen:needLen]) |
||||
|
readChan.readLen += n |
||||
|
if err != nil { |
||||
|
readChan.err = err |
||||
|
ch <- readChan |
||||
|
return |
||||
|
} |
||||
|
|
||||
|
if readChan.readLen == needLen { |
||||
|
break |
||||
|
} |
||||
|
} |
||||
|
ch <- readChan |
||||
|
}(p) |
||||
|
|
||||
|
select { |
||||
|
case <-time.After(time.Second * timeout): |
||||
|
return 0, fmt.Errorf("requestId: %s, readLen timeout, timeout is %d(second),need read:%d", "sr.Headers.Get(HTTPHeaderOssRequestID)", timeout, len(p)) |
||||
|
case result := <-ch: |
||||
|
return result.readLen, result.err |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
func bytesToInt(b []byte, ret interface{}) { |
||||
|
binBuf := bytes.NewBuffer(b) |
||||
|
binary.Read(binBuf, binary.BigEndian, ret) |
||||
|
} |
||||
|
|
||||
|
func checksum(b []byte, rec uint32, ftype string) error { |
||||
|
c := crc32.ChecksumIEEE(b) |
||||
|
if c != rec { |
||||
|
return fmt.Errorf("parse type: %v, checksum failed, cal: %v, rec: %v\n", ftype, c, rec) |
||||
|
} |
||||
|
return nil |
||||
|
} |
||||
Write
Preview
Loading…
Cancel
Save
Reference in new issue