From aefe67140fa07984b01c272eecb3bd20156aa0e3 Mon Sep 17 00:00:00 2001 From: "wanbiao.ye" Date: Thu, 29 Sep 2016 11:42:41 +0800 Subject: [PATCH 1/8] feature: resume upload --- README.md | 11 +++++ upyun/upyun-rest-api.go | 93 +++++++++++++++++++++++++++++++++++++++++ upyun/upyun.go | 87 +++++++++++++++++++++++++++++++++++++- upyun/upyun_test.go | 75 +++++++++++++++++++++++++++++++++ 4 files changed, 265 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a5e5cfe..f39850e 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ Table of Contents * [获取空间存储使用量](#获取空间存储使用量) * [创建目录](#创建目录) * [上传](#上传) + * [断点续传](#断点续传) * [下载](#下载) * [删除](#删除) * [获取文件信息](#获取文件信息) @@ -116,6 +117,16 @@ func (u *UpYun) Put(key string, value io.Reader, useMD5 bool, `key` 为 UPYUN 上的存储路径，`value` 既可以是文件，也可以是 `buffer`，`useMD5` 是否 MD5 校验，`headers` 自定义上传参数，除 [上传参数](https://docs.upyun.com/api/rest_api/#_4)，还可以设置 `Content-Length`，支持流式上传。流式上传需要指定 `Contnet-Length`，如需 MD5 校验，需要设置 `Content-MD5`。 + +#### 断点续传 +```go +func (u *UpYun) ResumePut(key string, value *os.File, useMD5 bool, + headers map[string]string, reporter ResumeReporter) (http.Header, error) +``` + +以断点续传方式上传文件，当且仅当文件在上传过程中遭遇网络故障时，等待 `$ResumeWaitSeconds(5)` 秒后，在失败断点处自动重试 `$ResumeRetryCount(3)` 次。参数 reporter 用于报告上传进度，可传入 ResumeReporterPrintln 打印进度，nil 忽略进度，或任何自行实现的 ResumeReporter 对象。 + + #### 下载 ```go diff --git a/upyun/upyun-rest-api.go b/upyun/upyun-rest-api.go index 31171a2..bd5e55e 100644 --- a/upyun/upyun-rest-api.go +++ b/upyun/upyun-rest-api.go @@ -8,12 +8,14 @@ import ( "fmt" "io" "io/ioutil" + "net" "net/http" URL "net/url" "os" "path" "strconv" "strings" + "time" ) // UPYUN REST API Client @@ -145,6 +147,97 @@ func (u *UpYun) Put(key string, value io.Reader, useMD5 bool, return rtHeaders, err } +// Put uploads file object to UPYUN File System part by part, +// and automatically retries when a network problem occurs +func (u *UpYun) ResumePut(key string, value *os.File, useMD5 bool, + headers map[string]string, reporter ResumeReporter) (http.Header, error) { + if headers == nil { + headers = make(map[string]string) + } + + fileinfo, err := value.Stat() + if err != nil { + return nil, err + } + + // If filesize < resumePartSizeLowerLimit, use UpYun.Put() instead + if fileinfo.Size() < resumePartSizeLowerLimit { + return u.Put(key, value, useMD5, headers) + } + + maxPartID := int(fileinfo.Size() / resumePartSize) + if fileinfo.Size()%resumePartSize == 0 { + maxPartID-- + } + + var resp http.Header + + for part := 0; part <= maxPartID; part++ { + + innerHeaders := make(map[string]string) + for k, v := range headers { + innerHeaders[k] = v + } + + innerHeaders["X-Upyun-Part-Id"] = strconv.Itoa(part) + switch part { + case 0: + innerHeaders["X-Upyun-Multi-Type"] = headers["Content-Type"] + innerHeaders["X-Upyun-Multi-Length"] = strconv.FormatInt(fileinfo.Size(), 10) + innerHeaders["X-Upyun-Multi-Stage"] = "initiate,upload" + innerHeaders["Content-Length"] = strconv.Itoa(resumePartSize) + case maxPartID: + innerHeaders["X-Upyun-Multi-Stage"] = "upload,complete" + innerHeaders["Content-Length"] = fmt.Sprintf("%v", fileinfo.Size()-int64(resumePartSize)*int64(part)) + if useMD5 { + md5Hash := md5.New() + value.Seek(0, 0) + io.Copy(md5Hash, value) + md5 := md5Hash.Sum(nil) + innerHeaders["X-Upyun-Multi-MD5"] = base64Str(md5) + } + default: + innerHeaders["X-Upyun-Multi-Stage"] = "upload" + innerHeaders["Content-Length"] = strconv.Itoa(resumePartSize) + } + + file, err := NewFragmentFile(value, int64(part)*int64(resumePartSize), resumePartSize) + if err != nil { + return resp, err + } + if useMD5 { + innerHeaders["Content-MD5"], _ = file.MD5() + } + + // Retry when get net error from UpYun.Put(), return error in other cases + for i := 0; i < ResumeRetryCount+1; i++ { + resp, err = u.Put(key, file, useMD5, innerHeaders) + if err == nil { + break + } + // Retry only get net error + _, ok := err.(net.Error) + if !ok { + return resp, err + } + time.Sleep(time.Second * time.Duration(ResumeWaitSeconds)) + file.Seek(0, 0) + if i == ResumeRetryCount { + return resp, err + } + } + if reporter != nil { + reporter(part, maxPartID) + } + + if part == 0 { + headers["X-Upyun-Multi-UUID"] = resp.Get("X-Upyun-Multi-Uuid") + } + } + + return resp, nil +} + // Get gets the specified file in UPYUN File System func (u *UpYun) Get(key string, value io.Writer) (int, error) { length, _, err := u.doRESTRequest("GET", key, "", nil, value) diff --git a/upyun/upyun.go b/upyun/upyun.go index c103cbd..37ec6bf 100644 --- a/upyun/upyun.go +++ b/upyun/upyun.go @@ -6,11 +6,13 @@ import ( "crypto/md5" "encoding/base64" "encoding/hex" + "errors" "fmt" "io" "net" "net/http" URL "net/url" + "os" "strconv" "strings" "time" @@ -25,11 +27,19 @@ const ( defaultChunkSize = 32 * 1024 // defaultConnectTimeout: connection timeout when connect to upyun endpoint defaultConnectTimeout = 60 + // resumePartSize is the size of each part for resume upload + resumePartSize = 1024 * 1024 + // resumePartSizeLowerLimit is the lowest file size limit for resume upload + resumePartSizeLowerLimit int64 = resumePartSize * 10 ) -// chunkSize: chunk size when copy var ( + // chunkSize: chunk size when copy chunkSize = defaultChunkSize + // ResumeRetry is the number of retries for resume upload + ResumeRetryCount = 3 + // ResumeWaitSeconds is the number of time to wait when net error occurs + ResumeWaitSeconds = 5 ) // Util functions @@ -172,3 +182,78 @@ func newFileInfo(arg interface{}) *FileInfo { return &fileInfo } } + +// ResumeReporter +type ResumeReporter func(int, int) + +// ResumeReporterPrintln is the simple ResumeReporter for test +func ResumeReporterPrintln(partID int, maxPartID int) { + fmt.Printf("resume test reporter: %v / %v\n", partID, maxPartID) +} + +// FragmentFile is like os.File, but only a part of file can be Read(). +// return io.EOF when cursor fetch the limit. +type FragmentFile struct { + offset int64 + limit int + cursor int + *os.File +} + +// NewFragmentFile returns new FragmentFile. +func NewFragmentFile(file *os.File, offset int64, limit int) (*FragmentFile, error) { + sizedfile := &FragmentFile{ + offset: offset, + limit: limit, + File: file, + } + _, err := sizedfile.Seek(0, 0) + if err != nil { + return nil, err + } + return sizedfile, nil +} + +// Seek likes os.File.Seek() +func (f *FragmentFile) Seek(offset int64, whence int) (ret int64, err error) { + switch whence { + case 0: + f.cursor = int(offset) + return f.File.Seek(f.offset+offset, 0) + default: + return 0, errors.New("whence muse be 0") + } +} + +// Read is just like os.File.Read but return io.EOF when catch sizedfile's limit +// or the end of file +func (f *FragmentFile) Read(b []byte) (n int, err error) { + if f.cursor >= f.limit { + return 0, io.EOF + } + n, err = f.File.Read(b) + if err != nil { + return n, err + } + if int(f.cursor)+n > f.limit { + n = f.limit - f.cursor + f.cursor = f.limit + return n, nil + } + f.cursor += n + return n, err +} + +// Close will not actually close FragmentFile +func (f *FragmentFile) Close() error { + return nil +} + +// MD5 returns md5 of the FragmentFile. +func (f *FragmentFile) MD5() (string, error) { + cursor := f.cursor + f.Seek(0, 0) + md5, _, err := md5sum(f) + f.Seek(int64(cursor), 0) + return md5, err +} diff --git a/upyun/upyun_test.go b/upyun/upyun_test.go index ed24818..10ae6e6 100644 --- a/upyun/upyun_test.go +++ b/upyun/upyun_test.go @@ -2,7 +2,9 @@ package upyun import ( "bytes" + "crypto/md5" "fmt" + "io" "os" "strings" "testing" @@ -177,6 +179,79 @@ func TestGetLargeList(t *testing.T) { } } +func TestResumeSmallFile(t *testing.T) { + file, err := os.Open(upload) + if err != nil { + t.Error(err) + } + defer file.Close() + + md5Hash := md5.New() + io.Copy(md5Hash, file) + md5 := fmt.Sprintf("%x", md5Hash.Sum(nil)) + file.Seek(0, 0) + + _, err = up.ResumePut(testPath+"/"+upload, file, true, map[string]string{"Content-Type": "text/plain"}, nil) + if err != nil { + t.Error(err) + } + + _, err = up.GetInfo(testPath + "/" + upload) + if err != nil { + t.Error(err) + } + + buf := bytes.NewBuffer(make([]byte, 0, 1024)) + up.Get(testPath+"/"+upload, buf) + + md5Hash.Reset() + io.Copy(md5Hash, buf) + + if fmt.Sprintf("%x", md5Hash.Sum(nil)) != md5 { + t.Error("MD5 is inconsistent") + } +} + +func TestResumeBigFile(t *testing.T) { + file, err := os.OpenFile("/tmp/bigfile", os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0666) + if err != nil { + t.Error(err) + } + defer os.Remove(file.Name()) + defer file.Close() + + for i := 0; i < 15*1024; i++ { + file.Write(bytes.Repeat([]byte("1"), 1024)) + } + file.Seek(0, 0) + + md5Hash := md5.New() + io.Copy(md5Hash, file) + md5 := fmt.Sprintf("%x", md5Hash.Sum(nil)) + file.Seek(0, 0) + + _, err = up.ResumePut(testPath+"/"+"bigfile", file, true, nil, ResumeReporterPrintln) + if err != nil { + t.Error(err) + } + + defer up.Delete(testPath + "/" + "bigfile") + + _, err = up.GetInfo(testPath + "/" + "bigfile") + if err != nil { + t.Error(err) + } + + buf := bytes.NewBuffer(make([]byte, 0, 1024)) + up.Get(testPath+"/"+"bigfile", buf) + + md5Hash.Reset() + io.Copy(md5Hash, buf) + if fmt.Sprintf("%x", md5Hash.Sum(nil)) != md5 { + t.Error("MD5 is inconsistent") + } +} + func TestDelete(t *testing.T) { // delete file path := testPath + "/" + upload From b0c3167cb618ccd7756f676ee58dcbf1fa8585c7 Mon Sep 17 00:00:00 2001 From: "wanbiao.ye" Date: Thu, 29 Sep 2016 14:03:30 +0800 Subject: [PATCH 2/8] bug fix --- upyun/upyun-rest-api.go | 8 ++++---- upyun/upyun.go | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/upyun/upyun-rest-api.go b/upyun/upyun-rest-api.go index bd5e55e..1d07a31 100644 --- a/upyun/upyun-rest-api.go +++ b/upyun/upyun-rest-api.go @@ -161,7 +161,7 @@ func (u *UpYun) ResumePut(key string, value *os.File, useMD5 bool, } // If filesize < resumePartSizeLowerLimit, use UpYun.Put() instead - if fileinfo.Size() < resumePartSizeLowerLimit { + if fileinfo.Size() < resumeFileSizeLowerLimit { return u.Put(key, value, useMD5, headers) } @@ -188,7 +188,7 @@ func (u *UpYun) ResumePut(key string, value *os.File, useMD5 bool, innerHeaders["Content-Length"] = strconv.Itoa(resumePartSize) case maxPartID: innerHeaders["X-Upyun-Multi-Stage"] = "upload,complete" - innerHeaders["Content-Length"] = fmt.Sprintf("%v", fileinfo.Size()-int64(resumePartSize)*int64(part)) + innerHeaders["Content-Length"] = fmt.Sprint(fileinfo.Size() - int64(resumePartSize)*int64(part)) if useMD5 { md5Hash := md5.New() value.Seek(0, 0) @@ -220,11 +220,11 @@ func (u *UpYun) ResumePut(key string, value *os.File, useMD5 bool, if !ok { return resp, err } - time.Sleep(time.Second * time.Duration(ResumeWaitSeconds)) - file.Seek(0, 0) if i == ResumeRetryCount { return resp, err } + time.Sleep(time.Second * time.Duration(ResumeWaitSeconds)) + file.Seek(0, 0) } if reporter != nil { reporter(part, maxPartID) diff --git a/upyun/upyun.go b/upyun/upyun.go index 37ec6bf..7e82029 100644 --- a/upyun/upyun.go +++ b/upyun/upyun.go @@ -29,8 +29,8 @@ const ( defaultConnectTimeout = 60 // resumePartSize is the size of each part for resume upload resumePartSize = 1024 * 1024 - // resumePartSizeLowerLimit is the lowest file size limit for resume upload - resumePartSizeLowerLimit int64 = resumePartSize * 10 + // resumeFileSizeLowerLimit is the lowest file size limit for resume upload + resumeFileSizeLowerLimit int64 = resumePartSize * 10 ) var ( @@ -221,7 +221,7 @@ func (f *FragmentFile) Seek(offset int64, whence int) (ret int64, err error) { f.cursor = int(offset) return f.File.Seek(f.offset+offset, 0) default: - return 0, errors.New("whence muse be 0") + return 0, errors.New("whence must be 0") } } From 702a695d07951fafbb4e003de301faa112d10e57 Mon Sep 17 00:00:00 2001 From: "wanbiao.ye" Date: Thu, 29 Sep 2016 14:05:40 +0800 Subject: [PATCH 3/8] update readme.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f39850e..257dd7f 100644 --- a/README.md +++ b/README.md @@ -124,7 +124,7 @@ func (u *UpYun) ResumePut(key string, value *os.File, useMD5 bool, headers map[string]string, reporter ResumeReporter) (http.Header, error) ``` -以断点续传方式上传文件，当且仅当文件在上传过程中遭遇网络故障时，等待 `$ResumeWaitSeconds(5)` 秒后，在失败断点处自动重试 `$ResumeRetryCount(3)` 次。参数 reporter 用于报告上传进度，可传入 ResumeReporterPrintln 打印进度，nil 忽略进度，或任何自行实现的 ResumeReporter 对象。 +以断点续传方式上传文件，当文件在上传过程中遭遇网络故障时，将等待 `$ResumeWaitSeconds(default 5)` 秒后，在失败断点处自动重试 `$ResumeRetryCount(default 3)` 次。参数 reporter 用于报告上传进度。 #### 下载 From 593f2ef36ef30b59955303ba5473ec879f348792 Mon Sep 17 00:00:00 2001 From: Mohanson Date: Thu, 29 Sep 2016 14:15:22 +0800 Subject: [PATCH 4/8] typos the simple ResumeReporter -> a simple ResumeReporter --- upyun/upyun.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/upyun/upyun.go b/upyun/upyun.go index 7e82029..d68177a 100644 --- a/upyun/upyun.go +++ b/upyun/upyun.go @@ -186,7 +186,7 @@ func newFileInfo(arg interface{}) *FileInfo { // ResumeReporter type ResumeReporter func(int, int) -// ResumeReporterPrintln is the simple ResumeReporter for test +// ResumeReporterPrintln is a simple ResumeReporter for test func ResumeReporterPrintln(partID int, maxPartID int) { fmt.Printf("resume test reporter: %v / %v\n", partID, maxPartID) } From bf04b9cbd7432dbecd3e1d18abeabcf363b3fbb4 Mon Sep 17 00:00:00 2001 From: "wanbiao.ye" Date: Thu, 29 Sep 2016 15:11:09 +0800 Subject: [PATCH 5/8] improve --- README.md | 2 +- upyun/upyun-http-core.go | 4 +- upyun/upyun-rest-api.go | 2 +- upyun/upyun-resume.go | 93 ++++++++++++++++++++++++++++++++++++++++ upyun/upyun.go | 93 ++-------------------------------------- 5 files changed, 101 insertions(+), 93 deletions(-) create mode 100644 upyun/upyun-resume.go diff --git a/README.md b/README.md index 257dd7f..a3cf9fb 100644 --- a/README.md +++ b/README.md @@ -124,7 +124,7 @@ func (u *UpYun) ResumePut(key string, value *os.File, useMD5 bool, headers map[string]string, reporter ResumeReporter) (http.Header, error) ``` -以断点续传方式上传文件，当文件在上传过程中遭遇网络故障时，将等待 `$ResumeWaitSeconds(default 5)` 秒后，在失败断点处自动重试 `$ResumeRetryCount(default 3)` 次。参数 reporter 用于报告上传进度。 +以断点续传方式上传文件，当文件在上传过程中遭遇网络故障时，将等待 `$ResumeWaitTime(default 5s)` 秒后，在失败断点处自动重试 `$ResumeRetryCount(default 3)` 次。参数 reporter 用于报告上传进度。 #### 下载 diff --git a/upyun/upyun-http-core.go b/upyun/upyun-http-core.go index cc2a974..bb6a34d 100644 --- a/upyun/upyun-http-core.go +++ b/upyun/upyun-http-core.go @@ -27,11 +27,11 @@ type upYunHTTPCore struct { httpClient *http.Client } -func (core *upYunHTTPCore) SetTimeout(timeout int) { +func (core *upYunHTTPCore) SetTimeout(timeout time.Duration) { core.httpClient = &http.Client{ Transport: &http.Transport{ Dial: func(network, addr string) (c net.Conn, err error) { - c, err = net.DialTimeout(network, addr, time.Duration(timeout)*time.Second) + c, err = net.DialTimeout(network, addr, timeout) if err != nil { return nil, err } diff --git a/upyun/upyun-rest-api.go b/upyun/upyun-rest-api.go index 1d07a31..b7001f2 100644 --- a/upyun/upyun-rest-api.go +++ b/upyun/upyun-rest-api.go @@ -223,7 +223,7 @@ func (u *UpYun) ResumePut(key string, value *os.File, useMD5 bool, if i == ResumeRetryCount { return resp, err } - time.Sleep(time.Second * time.Duration(ResumeWaitSeconds)) + time.Sleep(ResumeWaitTime) file.Seek(0, 0) } if reporter != nil { diff --git a/upyun/upyun-resume.go b/upyun/upyun-resume.go new file mode 100644 index 0000000..42fae87 --- /dev/null +++ b/upyun/upyun-resume.go @@ -0,0 +1,93 @@ +package upyun + +import ( + "errors" + "fmt" + "io" + "os" + "time" +) + +const ( + // resumePartSize is the size of each part for resume upload + resumePartSize = 1024 * 1024 + // resumeFileSizeLowerLimit is the lowest file size limit for resume upload + resumeFileSizeLowerLimit int64 = resumePartSize * 10 +) + +var ( + // ResumeRetry is the number of retries for resume upload + ResumeRetryCount = 3 + // ResumeWaitSeconds is the number of time to wait when net error occurs + ResumeWaitTime = time.Second * 5 +) + +// ResumeReporter +type ResumeReporter func(int, int) + +// ResumeReporterPrintln is a simple ResumeReporter for test +func ResumeReporterPrintln(partID int, maxPartID int) { + fmt.Printf("resume test reporter: %v / %v\n", partID, maxPartID) +} + +// FragmentFile is like os.File, but only a part of file can be Read(). +// return io.EOF when cursor fetch the limit. +type FragmentFile struct { + offset int64 + limit int + cursor int + *os.File +} + +// NewFragmentFile returns a new FragmentFile. +func NewFragmentFile(file *os.File, offset int64, limit int) (*FragmentFile, error) { + sizedfile := &FragmentFile{ + offset: offset, + limit: limit, + File: file, + } + _, err := sizedfile.Seek(0, 0) + if err != nil { + return nil, err + } + return sizedfile, nil +} + +// Seek likes os.File.Seek() +func (f *FragmentFile) Seek(offset int64, whence int) (ret int64, err error) { + switch whence { + case 0: + f.cursor = int(offset) + return f.File.Seek(f.offset+offset, 0) + default: + return 0, errors.New("whence must be 0") + } +} + +// Read is just like os.File.Read but return io.EOF when catch sizedfile's limit +// or the end of file +func (f *FragmentFile) Read(b []byte) (n int, err error) { + if f.cursor >= f.limit { + return 0, io.EOF + } + n, err = f.File.Read(b) + if int(f.cursor)+n > f.limit { + n = f.limit - f.cursor + } + f.cursor += n + return n, err +} + +// Close will not actually close FragmentFile +func (f *FragmentFile) Close() error { + return nil +} + +// MD5 returns md5 of the FragmentFile. +func (f *FragmentFile) MD5() (string, error) { + cursor := f.cursor + f.Seek(0, 0) + md5, _, err := md5sum(f) + f.Seek(int64(cursor), 0) + return md5, err +} diff --git a/upyun/upyun.go b/upyun/upyun.go index d68177a..8dfc5e2 100644 --- a/upyun/upyun.go +++ b/upyun/upyun.go @@ -6,13 +6,11 @@ import ( "crypto/md5" "encoding/base64" "encoding/hex" - "errors" "fmt" "io" "net" "net/http" URL "net/url" - "os" "strconv" "strings" "time" @@ -26,20 +24,12 @@ const ( // Default(Min/Max)ChunkSize: set the buffer size when doing copy operation defaultChunkSize = 32 * 1024 // defaultConnectTimeout: connection timeout when connect to upyun endpoint - defaultConnectTimeout = 60 - // resumePartSize is the size of each part for resume upload - resumePartSize = 1024 * 1024 - // resumeFileSizeLowerLimit is the lowest file size limit for resume upload - resumeFileSizeLowerLimit int64 = resumePartSize * 10 + defaultConnectTimeout = time.Second * 60 ) +// chunkSize: chunk size when copy var ( - // chunkSize: chunk size when copy chunkSize = defaultChunkSize - // ResumeRetry is the number of retries for resume upload - ResumeRetryCount = 3 - // ResumeWaitSeconds is the number of time to wait when net error occurs - ResumeWaitSeconds = 5 ) // Util functions @@ -120,9 +110,9 @@ func chunkedCopy(dst io.Writer, src io.Reader) (written int64, err error) { } // Use for http connection timeout -func timeoutDialer(timeout int) func(string, string) (net.Conn, error) { +func timeoutDialer(timeout time.Duration) func(string, string) (net.Conn, error) { return func(network, addr string) (c net.Conn, err error) { - c, err = net.DialTimeout(network, addr, time.Duration(timeout)*time.Second) + c, err = net.DialTimeout(network, addr, timeout) if err != nil { return nil, err } @@ -182,78 +172,3 @@ func newFileInfo(arg interface{}) *FileInfo { return &fileInfo } } - -// ResumeReporter -type ResumeReporter func(int, int) - -// ResumeReporterPrintln is a simple ResumeReporter for test -func ResumeReporterPrintln(partID int, maxPartID int) { - fmt.Printf("resume test reporter: %v / %v\n", partID, maxPartID) -} - -// FragmentFile is like os.File, but only a part of file can be Read(). -// return io.EOF when cursor fetch the limit. -type FragmentFile struct { - offset int64 - limit int - cursor int - *os.File -} - -// NewFragmentFile returns new FragmentFile. -func NewFragmentFile(file *os.File, offset int64, limit int) (*FragmentFile, error) { - sizedfile := &FragmentFile{ - offset: offset, - limit: limit, - File: file, - } - _, err := sizedfile.Seek(0, 0) - if err != nil { - return nil, err - } - return sizedfile, nil -} - -// Seek likes os.File.Seek() -func (f *FragmentFile) Seek(offset int64, whence int) (ret int64, err error) { - switch whence { - case 0: - f.cursor = int(offset) - return f.File.Seek(f.offset+offset, 0) - default: - return 0, errors.New("whence must be 0") - } -} - -// Read is just like os.File.Read but return io.EOF when catch sizedfile's limit -// or the end of file -func (f *FragmentFile) Read(b []byte) (n int, err error) { - if f.cursor >= f.limit { - return 0, io.EOF - } - n, err = f.File.Read(b) - if err != nil { - return n, err - } - if int(f.cursor)+n > f.limit { - n = f.limit - f.cursor - f.cursor = f.limit - return n, nil - } - f.cursor += n - return n, err -} - -// Close will not actually close FragmentFile -func (f *FragmentFile) Close() error { - return nil -} - -// MD5 returns md5 of the FragmentFile. -func (f *FragmentFile) MD5() (string, error) { - cursor := f.cursor - f.Seek(0, 0) - md5, _, err := md5sum(f) - f.Seek(int64(cursor), 0) - return md5, err -} From 82abdcaefcb1b5e9da2dfd1ad51bf7b3f0f6aca1 Mon Sep 17 00:00:00 2001 From: "wanbiao.ye" Date: Sun, 9 Oct 2016 10:22:25 +0800 Subject: [PATCH 6/8] Change value of X-Upyun-Multi-MD5 --- upyun/upyun-rest-api.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/upyun/upyun-rest-api.go b/upyun/upyun-rest-api.go index b7001f2..3bce0c6 100644 --- a/upyun/upyun-rest-api.go +++ b/upyun/upyun-rest-api.go @@ -190,11 +190,9 @@ func (u *UpYun) ResumePut(key string, value *os.File, useMD5 bool, innerHeaders["X-Upyun-Multi-Stage"] = "upload,complete" innerHeaders["Content-Length"] = fmt.Sprint(fileinfo.Size() - int64(resumePartSize)*int64(part)) if useMD5 { - md5Hash := md5.New() value.Seek(0, 0) - io.Copy(md5Hash, value) - md5 := md5Hash.Sum(nil) - innerHeaders["X-Upyun-Multi-MD5"] = base64Str(md5) + hex, _, _ := md5sum(value) + innerHeaders["X-Upyun-Multi-MD5"] = hex } default: innerHeaders["X-Upyun-Multi-Stage"] = "upload" From 987b07d64abede936a2942695020565238cd62df Mon Sep 17 00:00:00 2001 From: Mohanson Date: Tue, 11 Oct 2016 20:33:22 +0800 Subject: [PATCH 7/8] update doc --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a3cf9fb..ddbc7e8 100644 --- a/README.md +++ b/README.md @@ -124,7 +124,7 @@ func (u *UpYun) ResumePut(key string, value *os.File, useMD5 bool, headers map[string]string, reporter ResumeReporter) (http.Header, error) ``` -以断点续传方式上传文件，当文件在上传过程中遭遇网络故障时，将等待 `$ResumeWaitTime(default 5s)` 秒后，在失败断点处自动重试 `$ResumeRetryCount(default 3)` 次。参数 reporter 用于报告上传进度。 +以断点续传方式上传文件，当文件在上传过程中遭遇网络故障时，将等待 5 秒后，在失败断点处自动重试 3 次。参数 reporter 用于报告上传进度。可通过修改全局变量 `ResumeWaitTime` 与 `ResumeRetryCount` 自定义重试等待时间与重试次数。 #### 下载 From 169c1eaf78abdc58e8cb03b3734362f2f3b88786 Mon Sep 17 00:00:00 2001 From: Mohanson Date: Wed, 12 Oct 2016 10:56:55 +0800 Subject: [PATCH 8/8] Update README.md reporter => `reporter` --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ddbc7e8..0554eb1 100644 --- a/README.md +++ b/README.md @@ -124,7 +124,7 @@ func (u *UpYun) ResumePut(key string, value *os.File, useMD5 bool, headers map[string]string, reporter ResumeReporter) (http.Header, error) ``` -以断点续传方式上传文件，当文件在上传过程中遭遇网络故障时，将等待 5 秒后，在失败断点处自动重试 3 次。参数 reporter 用于报告上传进度。可通过修改全局变量 `ResumeWaitTime` 与 `ResumeRetryCount` 自定义重试等待时间与重试次数。 +以断点续传方式上传文件，当文件在上传过程中遭遇网络故障时，将等待 5 秒后，在失败断点处自动重试 3 次。参数 `reporter` 用于报告上传进度。可通过修改全局变量 `ResumeWaitTime` 与 `ResumeRetryCount` 自定义重试等待时间与重试次数。 #### 下载