diff --git a/.travis.yml b/.travis.yml index 315d014..b7c299c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,10 @@ language: go go: - - 1.3 - 1.4 - 1.5 + - 1.6 + - 1.7 install: - go get github.com/upyun/go-sdk/upyun diff --git a/README.md b/README.md index 0554eb1..05328c3 100644 --- a/README.md +++ b/README.md @@ -4,175 +4,153 @@ import "github.com/upyun/go-sdk/upyun" -UPYUN Go SDK, 集成： -- [UPYUN HTTP REST 接口](http://docs.upyun.com/api/rest_api/) -- [UPYUN HTTP FORM 接口](http://docs.upyun.com/api/form_api/) -- [UPYUN 缓存刷新接口](http://docs.upyun.com/api/purge/) -- [UPYUN 分块上传接口](http://docs.upyun.com/api/multipart_upload/) -- [UPYUN 视频处理接口](http://docs.upyun.com/api/av_pretreatment/) +又拍云 Go SDK, 集成： +- [又拍云 HTTP REST 接口](http://docs.upyun.com/api/rest_api/) +- [又拍云 HTTP FORM 接口](http://docs.upyun.com/api/form_api/) +- [又拍云缓存刷新接口](http://docs.upyun.com/api/purge/) +- [又拍云视频处理接口](http://docs.upyun.com/api/av_pretreatment/) Table of Contents ================= * [UPYUN Go SDK](#upyun-go-sdk) - * [Examples](#examples) * [Projects using this SDK](#projects-using-this-sdk) * [Usage](#usage) - * [UPYUN HTTP REST 接口](#upyun-http-rest-接口) - * [UpYun](#upyun) - * [初始化 UpYun](#初始化-upyun) - * [设置 API 访问域名](#设置-api-访问域名) + * [快速上手](#快速上手) + * [初始化 UpYun](#初始化-upyun) + * [又拍云 REST API 接口](#又拍云-rest-api-接口) * [获取空间存储使用量](#获取空间存储使用量) * [创建目录](#创建目录) * [上传](#上传) - * [断点续传](#断点续传) * [下载](#下载) * [删除](#删除) * [获取文件信息](#获取文件信息) * [获取文件列表](#获取文件列表) - * [UPYUN 缓存刷新接口](#upyun-缓存刷新接口) - * [UPYUN HTTP 表单上传接口](#upyun-http-表单上传接口) - * [UpYunForm](#upyunform) - * [初始化 UpYunForm](#初始化-upyunform) - * [FormAPIResp](#formapiresp) - * [设置 API 访问域名](#设置-api-访问域名-1) - * [上传文件](#上传文件) - * [UPYUN 分块上传接口](#upyun-分块上传接口) - * [UpYunMultiPart](#upyunmultipart) - * [UploadResp](#uploadresp) - * [MergeResp](#mergeresp) - * [初始化 UpYunMultiPart](#初始化-upyunmultipart) - * [上传](#上传-1) - * [UPYUN 音视频处理接口](#upyun-音视频处理接口) - * [UpYunMedia](#upyunmedia) - * [MediaStatusResp](#mediastatusresp) - * [初始化 UpYunMedia](#初始化-upyunmedia) - * [提交任务](#提交任务) - * [查询进度](#查询进度) - -## Examples - -示例代码见 `examples/`。 + * [又拍云缓存刷新接口](#又拍云缓存刷新接口) + * [又拍云表单上传接口](#又拍云表单上传接口) + * [又拍云处理接口](#又拍云处理接口) + * [提交处理任务](#提交处理任务) + * [获取处理进度](#获取处理进度) + * [获取处理结果](#获取处理结果) + * [基本类型](#基本类型) + * [UpYun](#upyun) + * [FileInfo](#fileinfo) + * [FormUploadResp](#formuploadresp) + * [PutObjectConfig](#putobjectconfig) + * [GetObjectConfig](#getobjectconfig) + * [GetObjectsConfig](#getobjectsconfig) + * [DeleteObjectConfig](#deleteobjectconfig) + * [FormUploadConfig](#formuploadconfig) + * [CommitTasksConfig](#committasksconfig) ## Projects using this SDK -- [UPYUN Command Tool](https://github.com/polym/upx) by [polym](https://github.com/polym) +- [又拍云命令行工具](https://github.com/polym/upx) by [polym](https://github.com/polym) ## Usage -### UPYUN HTTP REST 接口 - -#### UpYun +### 快速上手 ```go -type UpYun struct { - Bucket string // 空间名（即服务名称） - Username string // 操作员 - Passwd string // 密码 - ChunkSize int // 块读取大小, 默认32KB +package main + +import ( + "fmt" + "github.com/upyun/go-sdk/upyun" +) + +func main() { + up := upyun.NewUpYun(&upyun.UpYunConfig{ + Bucket: "demo", + Operator: "op", + Password: "password", + }) + + // 上传文件 + fmt.Println(up.Put(&upyun.PutObjectConfig{ + Path: "/demo.log", + LocalPath: "/tmp/upload", + })) + + // 下载 + fmt.Println(up.Get(&upyun.GetObjectConfig{ + Path: "/demo.log", + LocalPath: "/tmp/download", + })) + + // 列目录 + objsChan := make(chan *upyun.FileInfo, 10) + go func() { + fmt.Println(up.List(&upyun.GetObjectsConfig{ + Path: "/", + ObjectsChan: objsChan, + })) + }() + + for obj := range objsChan { + fmt.Println(obj) + } } ``` -#### 初始化 UpYun + +### 初始化 UpYun ```go -func NewUpYun(bucket, username, passwd string) *UpYun +func NewUpYun(config *UpYunConfig) *UpYun ``` -#### 设置 API 访问域名 +`NewUpYun` 初始化 `UpYun`，`UpYun` 是调用又拍云服务的统一入口，`UpYun` 对所有开放的接口都做了支持。 -```go -// Auto: Auto detected, based on user's internet -// Telecom: (ISP) China Telecom -// Cnc: (ISP) China Unicom -// Ctt: (ISP) China Tietong -const ( - Auto = iota - Telecom - Cnc - Ctt -) +--- -func (u *UpYun) SetEndpoint(ed int) error -``` +### 又拍云 REST API 接口 #### 获取空间存储使用量 ```go -func (u *UpYun) Usage() (int64, error) +func (up *UpYun) Usage() (n int64, err error) ``` #### 创建目录 ```go -func (u *UpYun) Mkdir(key string) error +func (up *UpYun) Mkdir(path string) error ``` #### 上传 ```go -func (u *UpYun) Put(key string, value io.Reader, useMD5 bool, - headers map[string]string) (http.Header, error) -``` - -`key` 为 UPYUN 上的存储路径，`value` 既可以是文件，也可以是 `buffer`，`useMD5` 是否 MD5 校验，`headers` 自定义上传参数，除 [上传参数](https://docs.upyun.com/api/rest_api/#_4)，还可以设置 `Content-Length`，支持流式上传。流式上传需要指定 `Contnet-Length`，如需 MD5 校验，需要设置 `Content-MD5`。 - - -#### 断点续传 -```go -func (u *UpYun) ResumePut(key string, value *os.File, useMD5 bool, - headers map[string]string, reporter ResumeReporter) (http.Header, error) +func (up *UpYun) Put(config *PutObjectConfig) (err error) ``` -以断点续传方式上传文件，当文件在上传过程中遭遇网络故障时，将等待 5 秒后，在失败断点处自动重试 3 次。参数 `reporter` 用于报告上传进度。可通过修改全局变量 `ResumeWaitTime` 与 `ResumeRetryCount` 自定义重试等待时间与重试次数。 - - #### 下载 ```go -func (u *UpYun) Get(key string, value io.Writer) (int, error) +func (up *UpYun) Get(config *GetObjectConfig) (fInfo *FileInfo, err error) ``` -此方法返回文件大小 - #### 删除 ```go -// 同步删除 -func (u *UpYun) Delete(key string) error - -// 异步删除文件 -func (u *UpYun) AsyncDelete(key string) error +func (up *UpYun) Delete(config *DeleteObjectConfig) error ``` #### 获取文件信息 ```go -type FileInfo struct { - Size int64 // 文件大小 - Time time.Time // 修改时间 - Name string // 文件名 - Type string // 类型，folder 或者 file -} - -func (u *UpYun) GetInfo(key string) (*FileInfo, error) +func (up *UpYun) GetInfo(path string) (*FileInfo, error) ``` #### 获取文件列表 ```go -// 少量文件 -func (u *UpYun) GetList(key string) ([]*FileInfo, error) - -// 大量文件 -func (u *UpYun) GetLargeList(key string, asc, recursive bool) (chan *FileInfo, chan error) +func (up *UpYun) List(config *GetObjectsConfig) error ``` -`key` 必须为目录。对于目录下有大量文件的，建议使用 `GetLargeList`。 - --- -### UPYUN 缓存刷新接口 +### 又拍云缓存刷新接口 ```go func (u *UpYun) Purge(urls []string) (string, error) @@ -180,154 +158,188 @@ func (u *UpYun) Purge(urls []string) (string, error) --- -### UPYUN HTTP 表单上传接口 - -#### UpYunForm +### 又拍云表单上传接口 ```go -type UpYunForm struct { - Secret string // 表单密钥 - Bucket string // 空间名（即服务名称） -} +func (up *UpYun) FormUpload(config *FormUploadConfig) (*FormUploadResp, error) ``` -#### 初始化 UpYunForm +--- -```go -func NewUpYunForm(bucket, key string) *UpYunForm -``` +### 又拍云处理接口 -#### FormAPIResp +#### 提交处理任务 ```go -type FormAPIResp struct { - Code int `json:"code"` - Msg string `json:"message"` - Url string `json:"url"` - Timestamp int64 `json:"time"` - ImgWidth int `json:"image-width"` - ImgHeight int `json:"image-height"` - ImgFrames int `json:"image-frames"` - ImgType string `json:"image-type"` - Sign string `json:"sign"` -} +func (up *UpYun) CommitTasks(config *CommitTasksConfig) (taskIds []string, err error) ``` -#### 设置 API 访问域名 +`tasksIds` 是提交任务的编号。通过这个编号，可以查询到处理进度以及处理结果等状态。 + +#### 获取处理进度 ```go -func (u *UpYunForm) SetEndpoint(ed int) error +func (up *UpYun) GetProgress(taskIds []string) (result map[string]int, err error) ``` -#### 上传文件 +#### 获取处理结果 ```go -func (uf *UpYunForm) Put(fpath, saveas string, expireAfter int64, - options map[string]string) (*FormAPIResp, error) +func (up *UpYun) GetResult(taskIds []string) (result map[string]interface{}, err error) ``` -`fpath` 上传文件名，`saveas` UPYUN 存储保存路径，`expireAfter` 过期时间长度，`options` 上传参数。 - --- -### UPYUN 分块上传接口 +### 基本类型 -#### UpYunMultiPart +#### UpYun ```go -type UpYunMultiPart struct { - Bucket string // 空间名（即服务名称） - Secret string // 表单密钥 - BlockSize int64 // 分块大小，单位字节, 建议 1024000 +type UpYunConfig struct { + Bucket string // 云存储服务名（空间名） + Operator string // 操作员 + Password string // 密码 + Secret string // 表单上传密钥，已经弃用！ + Hosts map[string]string // 自定义 Hosts 映射关系 + UserAgent string // HTTP User-Agent 头，默认 "UPYUN Go SDK V2" } ``` -#### UploadResp +`UpYunConfig` 提供初始化 `UpYun` 的所需参数。 需要注意的是，`Secret` 表单密钥已经弃用，如果一定需要使用，需调用 `UseDeprecatedApi`。 + + +#### FileInfo ```go -type UploadResp struct { - // returns after init request - SaveToken string `json:"save_token"` - // token_secert is equal to UPYUN Form API Secret - Secret string `json:"token_secret"` - // UPYUN Bucket Name - Bucket string `json:"bucket_name"` - // Number of Blocks - Blocks int `json:"blocks"` - Status []int `json:"status"` - ExpireAt int64 `json:"expire_at"` +type FileInfo struct { + Name string // 文件名 + Size int64 // 文件大小, 目录大小为 0 + ContentType string // 文件 Content-Type + IsDir bool // 是否为目录 + ETag string // ETag 值 + Time time.Time // 文件修改时间 + + Meta map[string]string // Metadata 数据 } ``` -#### MergeResp +#### FormUploadResp ```go -type MergeResp struct { - Path string `json:"path"` - ContentType string `json:"mimetype"` - ContentLength interface{} `json:"file_size"` - LastModify int64 `json:"last_modified"` - Signature string `json:"signature"` - ImageWidth int `json:"image_width"` - ImageHeight int `json:"image_height"` - ImageFrames int `json:"image_frames"` +type FormUploadResp struct { + Code int `json:"code"` // 状态码 + Msg string `json:"message"` // 状态信息 + Url string `json:"url"` // 保存路径 + Timestamp int64 `json:"time"` // 时间戳 + ImgWidth int `json:"image-width"` // 图片宽度 + ImgHeight int `json:"image-height"` // 图片高度 + ImgFrames int `json:"image-frames"` // 图片帧数 + ImgType string `json:"image-type"` // 图片类型 + Sign string `json:"sign"` // 签名 + Taskids []string `json:"task_ids"` // 异步任务 } ``` -#### 初始化 UpYunMultiPart +`FormUploadResp` 为表单上传的返回内容的格式。其中 `Code` 字段为状态码，可以查看 [API 错误码表](https://docs.upyun.com/api/errno/)。 + +#### PutObjectConfig ```go -func NewUpYunMultiPart(bucket, secret string, blocksize int64) *UpYunMultiPart +type PutObjectConfig struct { + Path string // 云存储中的路径 + LocalPath string // 待上传文件在本地文件系统中的路径 + Reader io.Reader // 待上传的内容 + Headers map[string]string // 额外的 HTTP 请求头 + UseMD5 bool // 是否需要 MD5 校验 + UseResumeUpload bool // 是否使用断点续传 + AppendContent bool // 是否需要追加文件内容 + ResumePartSize int64 // 断点续传块大小 + MaxResumePutTries int // 断点续传最大重试次数 +} ``` -#### 上传 +`PutObjectConfig` 提供上传单个文件所需的参数。有几点需要注意: +- `LocalPath` 跟 `Reader` 是互斥的关系，如果设置了 `LocalPath`，SDK 就会去读取这个文件，而忽略 `Reader` 中的内容。 +- 如果 `Reader` 是一个流／缓冲等的话，需要通过 `Headers` 参数设置 `Content-Length`，SDK 默认会对 `*os.File` 增加该字段。 +- [断点续传](https://docs.upyun.com/api/rest_api/#_3)的上传内容类型必须是 `*os.File`, 断点续传会将文件按照 `ResumePartSize` 进行切割，然后按次序一块一块上传，如果遇到网络问题，会进行重试，重试 `MaxResumePutTries` 次，默认无限重试。 +- `AppendContent` 如果是追加文件的话，确保非最后的分片必须为 1M 的整数倍。 +- 如果需要 MD5 校验，SDK 对 `*os.File` 会自动计算 MD5 值，其他类型需要自行通过 `Headers` 参数设置 `Content-MD5`。 + + +#### GetObjectConfig ```go -func (ump *UpYunMultiPart) Put(fpath, saveas string, - expireAfter int64, options map[string]interface{}) (*MergeResp, error) +type GetObjectConfig struct { + Path string // 云存储中的路径 + Headers map[string]string // 额外的 HTTP 请求头 + LocalPath string // 本地文件路径 + Writer io.Writer // 保存内容的容器 +} ``` ---- +`GetObjectConfig` 提供下载单个文件所需的参数。 跟 `PutObjectConfig` 类似，`LocalPath` 跟 `Writer` 是互斥的关系，如果设置了 `LocalPath`，SDK 就会把内容写入到这个文件中，而忽略 `Writer`。 -### UPYUN 音视频处理接口 -#### UpYunMedia +#### GetObjectsConfig ```go -type UpYunMedia struct { - Username string // 操作员 - Passwd string // 密码 - Bucket string // 空间名（即服务名称） +type GetObjectsConfig struct { + Path string // 云存储中的路径 + Headers map[string]string // 额外的 HTTP 请求头 + ObjectsChan chan *FileInfo // 对象通道 + QuitChan chan bool // 停止信号 + MaxListObjects int // 最大列对象个数 + MaxListTries int // 列目录最大重试次数 + MaxListLevel int // 递归最大深度 + DescOrder bool // 是否按降序列取，默认为升序 + + // Has unexported fields. } ``` -#### MediaStatusResp +`GetObjectsConfig` 提供列目录所需的参数。当列目录结束后，SDK 会将 `ObjectsChan` 关闭掉。 + + +#### DeleteObjectConfig ```go -type MediaStatusResp struct { - Tasks map[string]interface{} `json:"tasks"` +type DeleteObjectConfig struct { + Path string // 云存储中的路径 + Async bool // 是否使用异步删除 } ``` -#### 初始化 UpYunMedia +`DeleteObjectConfig` 提供删除单个文件／空目录所需的参数。 -```go -func NewUpYunMedia(bucket, user, pass string) *UpYunMedia -``` -#### 提交任务 +#### FormUploadConfig ```go -func (upm *UpYunMedia) PostTasks(src, notify, accept string, - tasks []map[string]interface{}) ([]string, error) +type FormUploadConfig struct { + LocalPath string // 待上传的文件路径 + SaveKey string // 保存路径 + ExpireAfterSec int64 // 签名超时时间 + NotifyUrl string // 结果回调地址 + Apps []map[string]interface{} // 异步处理任务 + Options map[string]interface{} // 更多自定义参数 +} ``` -`src` 音视频文件 UPYUN 存储路径，`notify` 回调URL，`accept` 设置回调格式，可选 `json`，`tasks` 任务列表，返回结果为任务 id 列表。 +`FormUploadConfig` 提供表单上传所需的参数。 + -#### 查询进度 +#### CommitTasksConfig ```go -func (upm *UpYunMedia) GetProgress(task_ids string) (*MediaStatusResp, error) +type CommitTasksConfig struct { + AppName string // 异步任务名称 + NotifyUrl string // 回调地址 + Tasks []interface{} // 任务数组 + + // Naga 相关配置 + Accept string // 回调支持的类型，默认为 json + Source string // 处理原文件路径 +} ``` -`task_ids` 是多个 `task_id` 用 `,` 连接起来。 +`CommitTasksConfig` 提供提交异步任务所需的参数。`Accept` 跟 `Source` 仅与异步音视频处理有关。`Tasks` 是一个任务数组，数组中的每一个元素都是任务相关的参数（一般情况下为字典类型）。 diff --git a/examples/cc.jpg b/examples/cc.jpg deleted file mode 100644 index c245254..0000000 Binary files a/examples/cc.jpg and /dev/null differ diff --git a/examples/config/config.go b/examples/config/config.go deleted file mode 100644 index ecf0630..0000000 --- a/examples/config/config.go +++ /dev/null @@ -1,9 +0,0 @@ -package config - -var ( - Bucket = "ServiceName" - Username = "OperatorName" - Passwd = "Password" - Notify = "http://www.upyun.com/notify" - Secret = "token secret in UPYUN Console" -) diff --git a/examples/upform.go b/examples/upform.go deleted file mode 100644 index e511d14..0000000 --- a/examples/upform.go +++ /dev/null @@ -1,24 +0,0 @@ -package main - -import ( - config "./config" - "fmt" - "github.com/upyun/go-sdk/upyun" -) - -func main() { - uf := upyun.NewUpYunForm(config.Bucket, config.Secret) - - options := map[string]string{ - "x-gmkerl-rotate": "90", - "notify-url": config.Notify, - } - fmt.Print(options) - - formResp, err := uf.Put("cc.jpg", "/{year}/{mon}/{day}/upload_{filename}{.suffix}", 3600, options) - if err != nil { - fmt.Println(err) - } else { - fmt.Printf("%+v\n", formResp) - } -} diff --git a/examples/upmedia.go b/examples/upmedia.go deleted file mode 100644 index d33ea4b..0000000 --- a/examples/upmedia.go +++ /dev/null @@ -1,29 +0,0 @@ -package main - -import ( - config "./config" - "fmt" - "github.com/upyun/go-sdk/upyun" - "strings" - "time" -) - -func main() { - upm := upyun.NewUpYunMedia(config.Bucket, config.Username, config.Passwd) - - task := map[string]interface{}{ - "type": "thumbnail", - "thumb_single": true, - } - tasks := []map[string]interface{}{task} - - ids, _ := upm.PostTasks("kai.3gp", config.Notify, "json", tasks) - - for { - status, _ := upm.GetProgress(strings.Join(ids, ",")) - for _, id := range ids { - fmt.Println(id, status.Tasks[id]) - } - time.Sleep(time.Second) - } -} diff --git a/examples/upmultipart.go b/examples/upmultipart.go deleted file mode 100644 index 51f347a..0000000 --- a/examples/upmultipart.go +++ /dev/null @@ -1,19 +0,0 @@ -package main - -import ( - config "./config" - "fmt" - "github.com/upyun/go-sdk/upyun" - "time" -) - -func main() { - ump := upyun.NewUpYunMultiPart(config.Bucket, config.Secret, 1024000) - options := map[string]interface{}{ - "x-gmkerl-rotate": "90", - "notify-url": config.Notify, - "ext-param": "123456", - } - resp, err := ump.Put("cc.jpg", "/test/IMG-c"+fmt.Sprint(time.Now().Unix())+".jpg", 3600, options) - fmt.Printf("%+v %v\n", resp, err) -} diff --git a/examples/uprest.go b/examples/uprest.go deleted file mode 100644 index ea3f625..0000000 --- a/examples/uprest.go +++ /dev/null @@ -1,36 +0,0 @@ -// SEE upyun_test.go -package main - -import ( - config "./config" - "fmt" - "github.com/upyun/go-sdk/upyun" - "os" - "time" -) - -func main() { - up := upyun.NewUpYun(config.Bucket, config.Username, config.Passwd) - headers := map[string]string{ - "x-gmkerl-watermark-type": "text", - "x-gmkerl-watermark-font": "simhei", - "x-gmkerl-watermark-color": "#faf1fb", - "x-gmkerl-watermark-size": "20", - "x-gmkerl-watermark-text": "UPYUN", - "x-gmkerl-watermark-border": "#40404085", - "x-gmkerl-watermark-margin": "10,10", - } - - fd, _ := os.Open("cc.jpg") - x := fmt.Sprintf("/wm/cc%d.jpg", time.Now().Unix()%10000) - fmt.Println(up.Put(x, fd, false, headers)) - - c := up.GetLargeList("/", true) - for { - v, more := <-c - if !more { - break - } - fmt.Println(v) - } -} diff --git a/upyun/auth.go b/upyun/auth.go new file mode 100644 index 0000000..6905ae9 --- /dev/null +++ b/upyun/auth.go @@ -0,0 +1,84 @@ +package upyun + +import ( + "fmt" + "sort" + "strings" +) + +type RESTAuthConfig struct { + Method string + Uri string + DateStr string + LengthStr string +} + +type PurgeAuthConfig struct { + PurgeList string + DateStr string +} + +type UnifiedAuthConfig struct { + Method string + Uri string + DateStr string + Policy string + ContentMD5 string +} + +func (u *UpYun) MakeRESTAuth(config *RESTAuthConfig) string { + sign := []string{ + config.Method, + config.Uri, + config.DateStr, + config.LengthStr, + u.Password, + } + return "UpYun " + u.Operator + ":" + md5Str(strings.Join(sign, "&")) +} + +func (u *UpYun) MakePurgeAuth(config *PurgeAuthConfig) string { + sign := []string{ + config.PurgeList, + u.Bucket, + config.DateStr, + u.Password, + } + return "UpYun " + u.Bucket + ":" + u.Operator + ":" + md5Str(strings.Join(sign, "&")) +} + +func (u *UpYun) MakeFormAuth(policy string) string { + return md5Str(base64ToStr([]byte(policy)) + "&" + u.Secret) +} + +func (u *UpYun) MakeProcessAuth(kwargs map[string]string) string { + keys := []string{} + for k, _ := range kwargs { + keys = append(keys, k) + } + sort.Strings(keys) + + auth := "" + for _, k := range keys { + auth += k + kwargs[k] + } + return fmt.Sprintf("UpYun %s:%s", u.Operator, md5Str(u.Operator+auth+u.Password)) +} + +func (u *UpYun) MakeUnifiedAuth(config *UnifiedAuthConfig) string { + sign := []string{ + config.Method, + config.Uri, + config.DateStr, + config.Policy, + config.ContentMD5, + } + signNoEmpty := []string{} + for _, v := range sign { + if v != "" { + signNoEmpty = append(signNoEmpty, v) + } + } + signStr := base64ToStr(hmacSha1(u.Password, []byte(strings.Join(signNoEmpty, "&")))) + return "UpYun " + u.Operator + ":" + signStr +} diff --git a/upyun/fileinfo.go b/upyun/fileinfo.go new file mode 100644 index 0000000..9d9d540 --- /dev/null +++ b/upyun/fileinfo.go @@ -0,0 +1,61 @@ +package upyun + +import ( + "net/http" + "strings" + "time" +) + +type FileInfo struct { + Name string + Size int64 + ContentType string + IsDir bool + ETag string + Time time.Time + + Meta map[string]string + + /* image information */ + ImgType string + ImgWidth int64 + ImgHeight int64 + ImgFrames int64 +} + +/* + Content-Type: image/gif + ETag: "dc9ea7257aa6da18e74505259b04a946" + x-upyun-file-type: GIF + x-upyun-height: 379 + x-upyun-width: 500 + x-upyun-frames: 90 +*/ +func parseHeaderToFileInfo(header http.Header, getinfo bool) *FileInfo { + fInfo := &FileInfo{} + for k, v := range header { + lk := strings.ToLower(k) + if strings.HasPrefix(lk, "x-upyun-meta-") { + if fInfo.Meta == nil { + fInfo.Meta = make(map[string]string) + } + fInfo.Meta[lk] = v[0] + } + } + + if getinfo { + // HTTP HEAD + fInfo.Size = parseStrToInt(header.Get("x-upyun-file-size")) + fInfo.IsDir = header.Get("x-upyun-file-type") == "folder" + fInfo.Time = time.Unix(parseStrToInt(header.Get("x-upyun-file-date")), 0) + } else { + fInfo.Size = parseStrToInt(header.Get("Content-Length")) + fInfo.ContentType = header.Get("Content-Type") + fInfo.ETag = header.Get("ETag") + fInfo.ImgType = header.Get("x-upyun-file-type") + fInfo.ImgWidth = parseStrToInt(header.Get("x-upyun-width")) + fInfo.ImgHeight = parseStrToInt(header.Get("x-upyun-height")) + fInfo.ImgFrames = parseStrToInt(header.Get("x-upyun-frames")) + } + return fInfo +} diff --git a/upyun/form.go b/upyun/form.go new file mode 100644 index 0000000..4b4517b --- /dev/null +++ b/upyun/form.go @@ -0,0 +1,144 @@ +package upyun + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "mime/multipart" + "net/http" + "os" + "path/filepath" + "time" +) + +type FormUploadConfig struct { + LocalPath string + SaveKey string + ExpireAfterSec int64 + NotifyUrl string + Apps []map[string]interface{} + Options map[string]interface{} +} + +type FormUploadResp struct { + Code int `json:"code"` + Msg string `json:"message"` + Url string `json:"url"` + Timestamp int64 `json:"time"` + ImgWidth int `json:"image-width"` + ImgHeight int `json:"image-height"` + ImgFrames int `json:"image-frames"` + ImgType string `json:"image-type"` + Sign string `json:"sign"` + Taskids []string `json:"task_ids"` +} + +func (config *FormUploadConfig) Format() { + if config.Options == nil { + config.Options = make(map[string]interface{}) + } + if config.SaveKey != "" { + config.Options["save-key"] = config.SaveKey + } + if config.NotifyUrl != "" { + config.Options["notify-url"] = config.NotifyUrl + } + if config.ExpireAfterSec > 0 { + config.Options["expiration"] = time.Now().Unix() + config.ExpireAfterSec + } + if len(config.Apps) > 0 { + config.Options["apps"] = config.Apps + } +} + +func (up *UpYun) FormUpload(config *FormUploadConfig) (*FormUploadResp, error) { + config.Format() + config.Options["bucket"] = up.Bucket + + args, err := json.Marshal(config.Options) + if err != nil { + return nil, err + } + policy := base64ToStr(args) + + formValues := make(map[string]string) + formValues["policy"] = policy + formValues["file"] = config.LocalPath + + if up.deprecated { + formValues["signature"] = up.MakeFormAuth(policy) + } else { + sign := &UnifiedAuthConfig{ + Method: "POST", + Uri: "/" + up.Bucket, + Policy: policy, + } + if v, ok := config.Options["date"]; ok { + sign.DateStr = v.(string) + } + if v, ok := config.Options["content-md5"]; ok { + sign.ContentMD5 = v.(string) + } + formValues["authorization"] = up.MakeUnifiedAuth(sign) + } + + endpoint := up.doGetEndpoint("v0.api.upyun.com") + url := fmt.Sprintf("http://%s/%s", endpoint, up.Bucket) + resp, err := up.doFormRequest(url, formValues) + if err != nil { + return nil, err + } + + b, _ := ioutil.ReadAll(resp.Body) + resp.Body.Close() + + if resp.StatusCode/100 != 2 { + return nil, fmt.Errorf("%s", string(b)) + } + + var r FormUploadResp + err = json.Unmarshal(b, &r) + return &r, err +} + +func (up *UpYun) doFormRequest(url string, formValues map[string]string) (*http.Response, error) { + formBody := &bytes.Buffer{} + formWriter := multipart.NewWriter(formBody) + defer formWriter.Close() + + for k, v := range formValues { + if k != "file" { + formWriter.WriteField(k, v) + } + } + + boundary := formWriter.Boundary() + bdBuf := bytes.NewBufferString(fmt.Sprintf("\r\n--%s--\r\n", boundary)) + + fpath := formValues["file"] + fd, err := os.Open(fpath) + if err != nil { + return nil, err + } + defer fd.Close() + + fInfo, err := fd.Stat() + if err != nil { + return nil, err + } + + _, err = formWriter.CreateFormFile("file", filepath.Base(fpath)) + if err != nil { + return nil, err + } + + headers := map[string]string{ + "Content-Type": "multipart/form-data; boundary=" + boundary, + "Content-Length": fmt.Sprint(formBody.Len() + int(fInfo.Size()) + bdBuf.Len()), + } + + body := io.MultiReader(formBody, fd, bdBuf) + return up.doHTTPRequest("POST", url, headers, body) +} diff --git a/upyun/form_test.go b/upyun/form_test.go new file mode 100644 index 0000000..0060c13 --- /dev/null +++ b/upyun/form_test.go @@ -0,0 +1,52 @@ +package upyun + +import ( + "path" + "testing" +) + +var ( + FORM_FILE = path.Join(ROOT, "FORM", "表单_FILE") +) + +func TestFormPutFile(t *testing.T) { + resp, err := up.FormUpload(&FormUploadConfig{ + LocalPath: LOCAL_FILE, + SaveKey: FORM_FILE, + ExpireAfterSec: 60, + }) + + Nil(t, err) + NotNil(t, resp) +} + +func TestFormPutApps(t *testing.T) { + thumb := map[string]interface{}{ + "name": "thumb", + "x-gmkerl-thumb": "/fw/120", + "save_as": "/x120.gif", + } + + naga := map[string]interface{}{ + "name": "naga", + "type": "video", + "avopts": "/f/mp4", + } + + spider := map[string]interface{}{ + "name": "spiderman", + "url": "http://www.upyun.com/index.html", + } + + resp, err := up.FormUpload(&FormUploadConfig{ + LocalPath: LOCAL_FILE, + SaveKey: FORM_FILE, + NotifyUrl: NOTIFY_URL, + ExpireAfterSec: 60, + Apps: []map[string]interface{}{thumb, naga, spider}, + }) + + NotNil(t, resp) + Nil(t, err) + Equal(t, len(resp.Taskids), 3) +} diff --git a/upyun/http.go b/upyun/http.go new file mode 100644 index 0000000..abb4cd1 --- /dev/null +++ b/upyun/http.go @@ -0,0 +1,55 @@ +package upyun + +import ( + // "fmt" + "io" + "net/http" + "os" + "strconv" + "strings" +) + +func (up *UpYun) doHTTPRequest(method, url string, headers map[string]string, + body io.Reader) (resp *http.Response, err error) { + req, err := http.NewRequest(method, url, body) + if err != nil { + return nil, err + } + + for k, v := range headers { + if strings.ToLower(k) == "host" { + req.Host = v + } else { + req.Header.Set(k, v) + } + } + + req.Header.Set("User-Agent", up.UserAgent) + if method == "PUT" || method == "POST" { + length := req.Header.Get("Content-Length") + if length != "" { + req.ContentLength, _ = strconv.ParseInt(length, 10, 64) + } else { + switch v := body.(type) { + case *os.File: + if fInfo, err := v.Stat(); err == nil { + req.ContentLength = fInfo.Size() + } + case UpYunPutReader: + req.ContentLength = int64(v.Len()) + } + } + } + + // fmt.Printf("%+v\n", req) + + return up.httpc.Do(req) +} + +func (up *UpYun) doGetEndpoint(host string) string { + s := up.Hosts[host] + if s != "" { + return s + } + return host +} diff --git a/upyun/io.go b/upyun/io.go new file mode 100644 index 0000000..b0b865f --- /dev/null +++ b/upyun/io.go @@ -0,0 +1,78 @@ +package upyun + +import ( + "fmt" + "io" + "os" +) + +type UpYunPutReader interface { + Len() (n int) + MD5() (ret string) + Read([]byte) (n int, err error) + Copyed() (n int) +} + +type fragmentFile struct { + realFile *os.File + offset int64 + limit int64 + cursor int64 +} + +func (f *fragmentFile) Seek(offset int64, whence int) (ret int64, err error) { + switch whence { + case 0: + f.cursor = offset + ret, err = f.realFile.Seek(f.offset+f.cursor, 0) + return ret - f.offset, err + default: + return 0, fmt.Errorf("whence must be 0") + } +} + +func (f *fragmentFile) Read(b []byte) (n int, err error) { + if f.cursor >= f.limit { + return 0, io.EOF + } + n, err = f.realFile.Read(b) + if f.cursor+int64(n) > f.limit { + n = int(f.limit - f.cursor) + } + f.cursor += int64(n) + return n, err +} + +func (f *fragmentFile) Stat() (fInfo os.FileInfo, err error) { + return fInfo, fmt.Errorf("fragmentFile not implement Stat()") +} + +func (f *fragmentFile) Close() error { + return nil +} + +func (f *fragmentFile) Copyed() int { + return int(f.cursor - f.offset) +} + +func (f *fragmentFile) Len() int { + return int(f.limit - f.offset) +} + +func (f *fragmentFile) MD5() string { + s, _ := md5File(f) + return s +} + +func newFragmentFile(file *os.File, offset, limit int64) (*fragmentFile, error) { + f := &fragmentFile{ + realFile: file, + offset: offset, + limit: limit, + } + + if _, err := f.Seek(0, 0); err != nil { + return nil, err + } + return f, nil +} diff --git a/upyun/process.go b/upyun/process.go new file mode 100644 index 0000000..ff976ad --- /dev/null +++ b/upyun/process.go @@ -0,0 +1,126 @@ +package upyun + +import ( + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "strings" + "time" +) + +type CommitTasksConfig struct { + AppName string + Accept string + Source string + NotifyUrl string + Tasks []interface{} +} + +func (up *UpYun) CommitTasks(config *CommitTasksConfig) (taskIds []string, err error) { + b, err := json.Marshal(config.Tasks) + if err != nil { + return nil, err + } + + kwargs := map[string]string{ + "app_name": config.AppName, + "tasks": base64ToStr(b), + "notify_url": config.NotifyUrl, + + // for naga + "source": config.Source, + } + if config.Accept != "" { + kwargs["accept"] = config.Accept + } + + err = up.doProcessRequest("POST", "/pretreatment/", kwargs, &taskIds) + return +} + +func (up *UpYun) GetProgress(taskIds []string) (result map[string]int, err error) { + kwargs := map[string]string{ + "task_ids": strings.Join(taskIds, ","), + } + v := map[string]map[string]int{} + err = up.doProcessRequest("GET", "/status/", kwargs, &v) + if err != nil { + return + } + + if r, ok := v["tasks"]; ok { + return r, err + } + return nil, fmt.Errorf("no tasks") +} + +func (up *UpYun) GetResult(taskIds []string) (result map[string]interface{}, err error) { + kwargs := map[string]string{ + "task_ids": strings.Join(taskIds, ","), + } + v := map[string]map[string]interface{}{} + err = up.doProcessRequest("GET", "/result/", kwargs, &v) + if err != nil { + return + } + + if r, ok := v["tasks"]; ok { + return r, err + } + return nil, fmt.Errorf("no tasks") +} + +func (up *UpYun) doProcessRequest(method, uri string, + kwargs map[string]string, v interface{}) error { + if _, ok := kwargs["service"]; !ok { + kwargs["service"] = up.Bucket + } + + if method == "GET" { + uri = addQueryToUri(uri, kwargs) + } + + headers := make(map[string]string) + headers["Date"] = makeRFC1123Date(time.Now()) + headers["Content-Type"] = "application/x-www-form-urlencoded" + if up.deprecated { + headers["Authorization"] = up.MakeProcessAuth(kwargs) + } else { + headers["Authorization"] = up.MakeUnifiedAuth(&UnifiedAuthConfig{ + Method: method, + Uri: uri, + DateStr: headers["Date"], + }) + } + + var resp *http.Response + var err error + endpoint := up.doGetEndpoint("p0.api.upyun.com") + rawurl := fmt.Sprintf("http://%s%s", endpoint, uri) + switch method { + case "GET": + resp, err = up.doHTTPRequest(method, rawurl, headers, nil) + case "POST": + payload := encodeQueryToPayload(kwargs) + resp, err = up.doHTTPRequest(method, rawurl, headers, bytes.NewBufferString(payload)) + default: + return fmt.Errorf("Unknown method") + } + + if err != nil { + return err + } + + b, err := ioutil.ReadAll(resp.Body) + resp.Body.Close() + if err != nil { + return err + } + if resp.StatusCode/100 != 2 { + return fmt.Errorf("%d %s", resp.StatusCode, string(b)) + } + + return json.Unmarshal(b, v) +} diff --git a/upyun/process_test.go b/upyun/process_test.go new file mode 100644 index 0000000..7e638fb --- /dev/null +++ b/upyun/process_test.go @@ -0,0 +1,61 @@ +package upyun + +import ( + "path" + "testing" +) + +var ( + MP4_URL = "http://prog-test.b0.upaiyun.com/const/kai.3gp" + MP4_SAVE_AS = path.Join(ROOT, "kai.3gp") + MP4_TASK_IDS []string +) + +func TestSpider(t *testing.T) { + task := map[string]interface{}{ + "url": MP4_URL, + "save_as": MP4_SAVE_AS, + } + ids, err := up.CommitTasks(&CommitTasksConfig{ + AppName: "spiderman", + NotifyUrl: NOTIFY_URL, + Tasks: []interface{}{task}, + }) + + Nil(t, err) + Equal(t, len(ids), 1) +} + +func TestNagaCommit(t *testing.T) { + task := map[string]interface{}{ + "type": "video", + "avopts": "/f/mp4", + } + task2 := map[string]interface{}{ + "type": "video", + "avopts": "/f/mp3", + } + + ids, err := up.CommitTasks(&CommitTasksConfig{ + AppName: "naga", + NotifyUrl: NOTIFY_URL, + Tasks: []interface{}{task, task2}, + }) + + Nil(t, err) + Equal(t, len(ids), 2) + + MP4_TASK_IDS = ids +} + +func TestNagaProgress(t *testing.T) { + res, err := up.GetProgress(MP4_TASK_IDS) + Nil(t, err) + Equal(t, len(res), 2) +} + +func TestNagaResult(t *testing.T) { + res, err := up.GetResult(MP4_TASK_IDS) + Nil(t, err) + Equal(t, len(res), 2) +} diff --git a/upyun/purge.go b/upyun/purge.go new file mode 100644 index 0000000..58dddef --- /dev/null +++ b/upyun/purge.go @@ -0,0 +1,58 @@ +package upyun + +import ( + "encoding/json" + "fmt" + "io/ioutil" + URL "net/url" + "strings" + "time" +) + +// TODO +func (up *UpYun) Purge(urls []string) (fails []string, err error) { + purge := "http://purge.upyun.com/purge/" + date := makeRFC1123Date(time.Now()) + purgeList := strings.Join(urls, "\n") + + headers := map[string]string{ + "Date": date, + "Authorization": up.MakePurgeAuth(&PurgeAuthConfig{ + PurgeList: purgeList, + DateStr: date, + }), + "Content-Type": "application/x-www-form-urlencoded;charset=utf-8", + } + + form := make(URL.Values) + form.Add("purge", purgeList) + + body := strings.NewReader(form.Encode()) + resp, err := up.doHTTPRequest("POST", purge, headers, body) + if err != nil { + return fails, err + } + defer resp.Body.Close() + + content, err := ioutil.ReadAll(resp.Body) + if err != nil { + return fails, err + } + + if resp.StatusCode/100 == 2 { + result := map[string]interface{}{} + if err := json.Unmarshal(content, &result); err != nil { + return fails, err + } + if it, ok := result["invalid_domain_of_url"]; ok { + if urls, ok := it.([]interface{}); ok { + for _, url := range urls { + fails = append(fails, url.(string)) + } + } + } + return fails, nil + } + + return nil, fmt.Errorf("purge %d %s", resp.StatusCode, string(content)) +} diff --git a/upyun/purge_test.go b/upyun/purge_test.go new file mode 100644 index 0000000..e5597e8 --- /dev/null +++ b/upyun/purge_test.go @@ -0,0 +1,24 @@ +package upyun + +import ( + "fmt" + "testing" +) + +func TestPurge(t *testing.T) { + fails, err := up.Purge([]string{ + fmt.Sprintf("http://%s.b0.upaiyun.com/demo.jpg", up.Bucket), + }) + + Nil(t, err) + Equal(t, len(fails), 0) + + fails, err = up.Purge([]string{ + fmt.Sprintf("http://%s.b0.upaiyun.com/demo.jpg", up.Bucket), + fmt.Sprintf("http://%s-t.b0.upaiyun.com/demo.jpg", up.Bucket), + }) + + Nil(t, err) + Equal(t, len(fails), 1) + Equal(t, fails[0], fmt.Sprintf("http://%s-t.b0.upaiyun.com/demo.jpg", up.Bucket)) +} diff --git a/upyun/rest.go b/upyun/rest.go new file mode 100644 index 0000000..b7e596e --- /dev/null +++ b/upyun/rest.go @@ -0,0 +1,496 @@ +package upyun + +import ( + "fmt" + "io" + "io/ioutil" + "net" + "net/http" + "os" + "path" + "strings" + "time" +) + +const ( + defaultResumePartSize = 1024 * 1024 + minResumePutFileSize = 10 * 1024 * 1024 +) + +type restReqConfig struct { + method string + uri string + headers map[string]string + closeBody bool + httpBody io.Reader + useMD5 bool +} + +// GetObjectConfig provides a configuration to Get method. +type GetObjectConfig struct { + Path string + // Headers contains custom http header, like User-Agent. + Headers map[string]string + LocalPath string + Writer io.Writer +} + +// GetObjectConfig provides a configuration to List method. +type GetObjectsConfig struct { + Path string + Headers map[string]string + ObjectsChan chan *FileInfo + QuitChan chan bool + MaxListObjects int + MaxListTries int + // MaxListLevel: depth of recursion + MaxListLevel int + // DescOrder: whether list objects by desc-order + DescOrder bool + + rootDir string + level int + objNum int + try int +} + +// PutObjectConfig provides a configuration to Put method. +type PutObjectConfig struct { + Path string + LocalPath string + Reader io.Reader + Headers map[string]string + UseMD5 bool + UseResumeUpload bool + AppendContent bool + ResumePartSize int64 + MaxResumePutTries int +} + +type DeleteObjectConfig struct { + Path string + Async bool +} + +type ModifyMetadataConfig struct { + Path string + Operation string + Headers map[string]string +} + +func (up *UpYun) Usage() (n int64, err error) { + var resp *http.Response + resp, err = up.doRESTRequest(&restReqConfig{ + method: "GET", + uri: "/?usage", + }) + + if err == nil { + n, err = readHTTPBodyToInt(resp) + } + + if err != nil { + return 0, fmt.Errorf("usage: %v", err) + } + return n, nil +} + +func (up *UpYun) Mkdir(path string) error { + _, err := up.doRESTRequest(&restReqConfig{ + method: "POST", + uri: path, + headers: map[string]string{ + "folder": "true", + "x-upyun-folder": "true", + }, + closeBody: true, + }) + if err != nil { + return fmt.Errorf("mkdir %s: %v", path, err) + } + return nil +} + +// TODO: maybe directory +func (up *UpYun) Get(config *GetObjectConfig) (fInfo *FileInfo, err error) { + if config.LocalPath != "" { + var fd *os.File + if fd, err = os.Create(config.LocalPath); err != nil { + return nil, fmt.Errorf("create file: %v", err) + } + defer fd.Close() + config.Writer = fd + } + + if config.Writer == nil { + return nil, fmt.Errorf("no writer") + } + + resp, err := up.doRESTRequest(&restReqConfig{ + method: "GET", + uri: config.Path, + }) + if err != nil { + return nil, fmt.Errorf("doRESTRequest: %v", err) + } + defer resp.Body.Close() + + fInfo = parseHeaderToFileInfo(resp.Header, false) + fInfo.Name = config.Path + + if fInfo.Size, err = io.Copy(config.Writer, resp.Body); err != nil { + return nil, fmt.Errorf("io copy: %v", err) + } + + return +} + +func (up *UpYun) put(config *PutObjectConfig) error { + if config.AppendContent { + if config.Headers == nil { + config.Headers = make(map[string]string) + } + config.Headers["X-Upyun-Append"] = "true" + } + _, err := up.doRESTRequest(&restReqConfig{ + method: "PUT", + uri: config.Path, + headers: config.Headers, + closeBody: true, + httpBody: config.Reader, + useMD5: config.UseMD5, + }) + if err != nil { + return fmt.Errorf("doRESTRequest: %v", err) + } + return nil +} + +// TODO: progress +func (up *UpYun) resumePut(config *PutObjectConfig) error { + f, ok := config.Reader.(*os.File) + if !ok { + return fmt.Errorf("resumePut: type != *os.File") + } + + fileinfo, err := f.Stat() + if err != nil { + return fmt.Errorf("Stat: %v", err) + } + + fsize := fileinfo.Size() + if fsize < minResumePutFileSize { + return up.put(config) + } + + if config.ResumePartSize == 0 { + config.ResumePartSize = defaultResumePartSize + } + maxPartID := int((fsize+config.ResumePartSize-1)/config.ResumePartSize - 1) + + if config.Headers == nil { + config.Headers = make(map[string]string) + } + + curSize, partSize := int64(0), config.ResumePartSize + headers := config.Headers + for id := 0; id <= maxPartID; id++ { + if curSize+partSize > fsize { + partSize = fsize - curSize + } + headers["Content-Length"] = fmt.Sprint(partSize) + headers["X-Upyun-Part-ID"] = fmt.Sprint(id) + + switch id { + case 0: + headers["X-Upyun-Multi-Type"] = headers["Content-Type"] + headers["X-Upyun-Multi-Length"] = fmt.Sprint(fsize) + headers["X-Upyun-Multi-Stage"] = "initiate,upload" + case int(maxPartID): + headers["X-Upyun-Multi-Stage"] = "upload,complete" + if config.UseMD5 { + f.Seek(0, 0) + headers["X-Upyun-Multi-MD5"], _ = md5File(f) + } + default: + headers["X-Upyun-Multi-Stage"] = "upload" + } + + fragFile, err := newFragmentFile(f, curSize, partSize) + if err != nil { + return fmt.Errorf("newFragmentFile: %v", err) + } + + try := 0 + var resp *http.Response + for ; config.MaxResumePutTries == 0 || try < config.MaxResumePutTries; try++ { + resp, err = up.doRESTRequest(&restReqConfig{ + method: "PUT", + uri: config.Path, + headers: headers, + closeBody: true, + useMD5: config.UseMD5, + httpBody: fragFile, + }) + if err == nil { + break + } + if _, ok := err.(net.Error); !ok { + return fmt.Errorf("doRESTRequest: %v", err) + } + fragFile.Seek(0, 0) + } + + if config.MaxResumePutTries > 0 && try == config.MaxResumePutTries { + return err + } + + if id == 0 { + headers["X-Upyun-Multi-UUID"] = resp.Header.Get("X-Upyun-Multi-UUID") + } else { + if id == maxPartID { + return nil + } + } + + curSize += partSize + } + + return nil +} + +func (up *UpYun) Put(config *PutObjectConfig) (err error) { + if config.LocalPath != "" { + var fd *os.File + if fd, err = os.Open(config.LocalPath); err != nil { + return fmt.Errorf("open file: %v", err) + } + defer fd.Close() + config.Reader = fd + } + + if config.UseResumeUpload { + return up.resumePut(config) + } + return up.put(config) +} + +func (up *UpYun) Delete(config *DeleteObjectConfig) error { + headers := map[string]string{} + if config.Async == true { + headers["x-upyun-async"] = "true" + } + _, err := up.doRESTRequest(&restReqConfig{ + method: "DELETE", + uri: config.Path, + headers: headers, + closeBody: true, + }) + if err != nil { + return fmt.Errorf("delete %s: %v", config.Path, err) + } + return nil +} + +func (up *UpYun) GetInfo(path string) (*FileInfo, error) { + resp, err := up.doRESTRequest(&restReqConfig{ + method: "HEAD", + uri: path, + closeBody: true, + }) + if err != nil { + return nil, fmt.Errorf("getinfo %s: %v", path, err) + } + fInfo := parseHeaderToFileInfo(resp.Header, true) + fInfo.Name = path + return fInfo, nil +} + +func (up *UpYun) List(config *GetObjectsConfig) error { + if config.ObjectsChan == nil { + return fmt.Errorf("ObjectsChan == nil") + } + if config.Headers == nil { + config.Headers = make(map[string]string) + } + if config.QuitChan == nil { + config.QuitChan = make(chan bool) + } + // 50 is nice value + if _, exist := config.Headers["X-List-Limit"]; !exist { + config.Headers["X-List-Limit"] = "50" + } + + if config.DescOrder { + config.Headers["X-List-Order"] = "desc" + } + + config.Headers["X-UpYun-Folder"] = "true" + + // 1st level + if config.level == 0 { + defer close(config.ObjectsChan) + } + + for { + resp, err := up.doRESTRequest(&restReqConfig{ + method: "GET", + uri: config.Path, + headers: config.Headers, + }) + + if err != nil { + if _, ok := err.(net.Error); ok { + config.try++ + if config.MaxListTries == 0 || config.try < config.MaxListTries { + continue + } + } + return err + } + + b, err := ioutil.ReadAll(resp.Body) + resp.Body.Close() + if err != nil { + return fmt.Errorf("ioutil ReadAll: %v", err) + } + + for _, fInfo := range parseBodyToFileInfos(b) { + if fInfo.IsDir && (config.level+1 < config.MaxListLevel || config.MaxListLevel == -1) { + rConfig := &GetObjectsConfig{ + Path: path.Join(config.Path, fInfo.Name), + QuitChan: config.QuitChan, + ObjectsChan: config.ObjectsChan, + MaxListTries: config.MaxListTries, + MaxListObjects: config.MaxListObjects, + DescOrder: config.DescOrder, + MaxListLevel: config.MaxListLevel, + level: config.level + 1, + rootDir: path.Join(config.rootDir, fInfo.Name), + try: config.try, + objNum: config.objNum, + } + if err = up.List(rConfig); err != nil { + return err + } + config.try, config.objNum = rConfig.try, rConfig.objNum + } + if config.rootDir != "" { + fInfo.Name = path.Join(config.rootDir, fInfo.Name) + } + + select { + case <-config.QuitChan: + return nil + default: + config.ObjectsChan <- fInfo + } + + config.objNum++ + if config.MaxListObjects > 0 && config.objNum >= config.MaxListObjects { + return nil + } + + } + + config.Headers["X-List-Iter"] = resp.Header.Get("X-Upyun-List-Iter") + if config.Headers["X-List-Iter"] == "g2gCZAAEbmV4dGQAA2VvZg" { + return nil + } + } +} + +func (up *UpYun) ModifyMetadata(config *ModifyMetadataConfig) error { + if config.Operation == "" { + config.Operation = "merge" + } + _, err := up.doRESTRequest(&restReqConfig{ + method: "PATCH", + uri: config.Path + "?metadata=" + config.Operation, + headers: config.Headers, + closeBody: true, + }) + return err +} + +func (up *UpYun) doRESTRequest(config *restReqConfig) (*http.Response, error) { + escUri, err := escapeUri(config.uri) + if err != nil { + return nil, err + } + escUri = path.Join("/", up.Bucket, escUri) + if strings.HasSuffix(config.uri, "/") { + escUri += "/" + } + + headers := map[string]string{} + hasMD5 := false + for k, v := range config.headers { + if strings.ToLower(k) == "content-md5" && v != "" { + hasMD5 = true + } + headers[k] = v + } + + headers["Date"] = makeRFC1123Date(time.Now()) + headers["Host"] = "v0.api.upyun.com" + + if !hasMD5 && config.useMD5 { + switch v := config.httpBody.(type) { + case *os.File: + headers["Content-MD5"], _ = md5File(v) + case UpYunPutReader: + headers["Content-MD5"] = v.MD5() + } + } + + if up.deprecated { + if _, ok := headers["Content-Length"]; !ok { + size := int64(0) + switch v := config.httpBody.(type) { + case *os.File: + if fInfo, err := v.Stat(); err == nil { + size = fInfo.Size() + } + case UpYunPutReader: + size = int64(v.Len()) + } + headers["Content-Length"] = fmt.Sprint(size) + } + headers["Authorization"] = up.MakeRESTAuth(&RESTAuthConfig{ + Method: config.method, + Uri: escUri, + DateStr: headers["Date"], + LengthStr: headers["Content-Length"], + }) + } else { + headers["Authorization"] = up.MakeUnifiedAuth(&UnifiedAuthConfig{ + Method: config.method, + Uri: escUri, + DateStr: headers["Date"], + ContentMD5: headers["Content-MD5"], + }) + } + + endpoint := up.doGetEndpoint("v0.api.upyun.com") + url := fmt.Sprintf("http://%s%s", endpoint, escUri) + + resp, err := up.doHTTPRequest(config.method, url, headers, config.httpBody) + if err != nil { + // Don't modify net error + return nil, err + } + + if resp.StatusCode/100 != 2 { + body, _ := ioutil.ReadAll(resp.Body) + resp.Body.Close() + return resp, fmt.Errorf("%s %d %s", config.method, resp.StatusCode, string(body)) + } + + if config.closeBody { + io.Copy(ioutil.Discard, resp.Body) + resp.Body.Close() + } + + return resp, nil +} diff --git a/upyun/rest_test.go b/upyun/rest_test.go new file mode 100644 index 0000000..88ec7cc --- /dev/null +++ b/upyun/rest_test.go @@ -0,0 +1,222 @@ +package upyun + +import ( + "bytes" + "fmt" + "io/ioutil" + "os" + "path" + "sort" + "strings" + "testing" +) + +var ( + REST_DIR = path.Join(ROOT, "REST") + REST_FILE_1 = path.Join(REST_DIR, "FILE_1") + REST_FILE_BUF = path.Join(REST_DIR, "FILE_BUF") + REST_FILE_1M = path.Join(REST_DIR, "FILE_1M") + REST_FILE_BUF_BUF = path.Join(REST_DIR, "文件_BUF_BUF") + REST_OBJS = []string{"FILE_1", "FILE_1M", "FILE_BUF", "文件_BUF_BUF"} + + BUF_CONTENT = "UPYUN GO SDK" + LOCAL_FILE = "./rest.go" + LOCAL_SAVE_FILE = LOCAL_FILE + "_bak" +) + +func TestUsage(t *testing.T) { + n, err := up.Usage() + Nil(t, err) + Equal(t, n > 0, true) +} + +func TestGetInfoDir(t *testing.T) { + fInfo, err := up.GetInfo("/") + Nil(t, err) + NotNil(t, fInfo) + Equal(t, fInfo.IsDir, true) +} + +func TestMkdir(t *testing.T) { + err := up.Mkdir(REST_DIR) + Nil(t, err) +} + +func TestPutWithFileReader(t *testing.T) { + fd, _ := os.Open(LOCAL_FILE) + NotNil(t, fd) + defer fd.Close() + + err := up.Put(&PutObjectConfig{ + Path: REST_FILE_1, + Reader: fd, + Headers: map[string]string{ + "X-Upyun-Meta-Filename": LOCAL_FILE, + }, + UseMD5: true, + }) + Nil(t, err) +} + +func TestPutWithBuffer(t *testing.T) { + s := BUF_CONTENT + r := strings.NewReader(s) + + err := up.Put(&PutObjectConfig{ + Path: REST_FILE_BUF, + Reader: r, + Headers: map[string]string{ + "Content-Length": fmt.Sprint(len(s)), + }, + UseMD5: true, + }) + Nil(t, err) +} + +func TestPutWithBufferAppend(t *testing.T) { + s := BUF_CONTENT + for k := 0; k < 3; k++ { + r := strings.NewReader(s) + err := up.Put(&PutObjectConfig{ + Path: REST_FILE_BUF_BUF, + Reader: r, + Headers: map[string]string{ + "Content-Length": fmt.Sprint(len(s)), + }, + AppendContent: true, + UseMD5: true, + }) + if k != 0 { + NotNil(t, err) + } else { + Nil(t, err) + } + } +} + +func TestResumePut(t *testing.T) { + fname := "1M" + fd, _ := os.Create(fname) + NotNil(t, fd) + kb := strings.Repeat("U", 1024) + for i := 0; i < (minResumePutFileSize/1024 + 2); i++ { + fd.WriteString(kb) + } + fd.Close() + + defer os.RemoveAll(fname) + + err := up.Put(&PutObjectConfig{ + Path: REST_FILE_1M, + LocalPath: fname, + UseMD5: true, + UseResumeUpload: true, + }) + Nil(t, err) +} + +func TestGetWithWriter(t *testing.T) { + b := make([]byte, 0) + buf := bytes.NewBuffer(b) + fInfo, err := up.Get(&GetObjectConfig{ + Path: REST_FILE_BUF, + Writer: buf, + }) + Nil(t, err) + NotNil(t, fInfo) + Equal(t, fInfo.IsDir, false) + Equal(t, fInfo.Size, int64(len(BUF_CONTENT))) + Equal(t, buf.String(), BUF_CONTENT) +} + +func TestGetWithLocalPath(t *testing.T) { + defer os.Remove(LOCAL_SAVE_FILE) + fInfo, err := up.Get(&GetObjectConfig{ + Path: REST_FILE_1, + LocalPath: LOCAL_SAVE_FILE, + }) + Nil(t, err) + NotNil(t, fInfo) + Equal(t, fInfo.IsDir, false) + + NotNil(t, fInfo.Meta) + name, _ := fInfo.Meta["x-upyun-meta-filename"] + Equal(t, name, LOCAL_FILE) + + _, err = os.Stat(LOCAL_SAVE_FILE) + Nil(t, err) + + b1, err := ioutil.ReadFile(LOCAL_FILE) + Nil(t, err) + + b2, err := ioutil.ReadFile(LOCAL_SAVE_FILE) + Nil(t, err) + + Equal(t, string(b1), string(b2)) +} + +func TestGetInfoFile(t *testing.T) { + fInfo, err := up.GetInfo(REST_FILE_BUF_BUF) + Nil(t, err) + NotNil(t, fInfo) + Equal(t, fInfo.IsDir, false) + Equal(t, fInfo.Name, REST_FILE_BUF_BUF) + // as append interface + Equal(t, fInfo.Size, int64(len(BUF_CONTENT))) +} + +func TestList(t *testing.T) { + ch := make(chan *FileInfo, 10) + files := []string{} + + go func() { + err := up.List(&GetObjectsConfig{ + Path: REST_DIR, + ObjectsChan: ch, + }) + Nil(t, err) + }() + + for fInfo := range ch { + files = append(files, fInfo.Name) + } + + Equal(t, len(files), len(REST_OBJS)) + sort.Strings(files) + sort.Strings(REST_OBJS) + for k := range REST_OBJS { + Equal(t, REST_OBJS[k], files[k]) + } +} + +func TestModifyMetadata(t *testing.T) { + // time.Sleep(10 * time.Second) + err := up.ModifyMetadata(&ModifyMetadataConfig{ + Path: REST_FILE_1, + Operation: "replace", + Headers: map[string]string{ + "X-Upyun-Meta-Filename": LOCAL_SAVE_FILE, + }, + }) + + Nil(t, err) +} + +func TestDelete(t *testing.T) { + err := up.Delete(&DeleteObjectConfig{ + Path: REST_DIR, + }) + NotNil(t, err) + + for _, obj := range REST_OBJS { + err := up.Delete(&DeleteObjectConfig{ + Path: path.Join(REST_DIR, obj), + }) + Nil(t, err) + } + + err = up.Delete(&DeleteObjectConfig{ + Path: REST_DIR, + }) + Nil(t, err) +} diff --git a/upyun/upyun-form-api.go b/upyun/upyun-form-api.go deleted file mode 100644 index d4d37f6..0000000 --- a/upyun/upyun-form-api.go +++ /dev/null @@ -1,110 +0,0 @@ -package upyun - -import ( - "encoding/base64" - "encoding/json" - "errors" - "fmt" - "io/ioutil" - "net/http" - "os" - "strconv" - "time" -) - -// UPYUN HTTP FORM API Client -type UpYunForm struct { - // Core - upYunHTTPCore - - Secret string - Bucket string -} - -// Response from UPYUN Form API Server -type FormAPIResp struct { - Code int `json:"code"` - Msg string `json:"message"` - Url string `json:"url"` - Timestamp int64 `json:"time"` - ImgWidth int `json:"image-width"` - ImgHeight int `json:"image-height"` - ImgFrames int `json:"image-frames"` - ImgType string `json:"image-type"` - Sign string `json:"sign"` -} - -// NewUpYunForm return a UPYUN Form API client given -// a form api key and bucket name. As Default, endpoint -// is set to Auto, http client connection timeout is -// set to defalutConnectionTimeout which is equal to -// 60 seconds. -func NewUpYunForm(bucket, secret string) *UpYunForm { - upm := &UpYunForm{ - Secret: secret, - Bucket: bucket, - } - - upm.httpClient = &http.Client{} - upm.SetTimeout(defaultConnectTimeout) - upm.SetEndpoint(Auto) - - return upm -} - -// SetEndpoint sets the request endpoint to UPYUN Form API Server. -func (u *UpYunForm) SetEndpoint(ed int) error { - if ed >= Auto && ed <= Ctt { - u.endpoint = fmt.Sprintf("v%d.api.upyun.com", ed) - return nil - } - - return errors.New("Invalid endpoint, pick from Auto, Telecom, Cnc, Ctt") -} - -// Put posts a http form request given reader, save path, -// expiration, other options and returns a FormAPIResp pointer. -func (uf *UpYunForm) Put(fpath, saveas string, expireAfter int64, - options map[string]string) (*FormAPIResp, error) { - if options == nil { - options = make(map[string]string) - } - - options["bucket"] = uf.Bucket - options["save-key"] = saveas - options["expiration"] = strconv.FormatInt(time.Now().Unix()+expireAfter, 10) - - args, err := json.Marshal(options) - if err != nil { - return nil, err - } - - policy := base64.StdEncoding.EncodeToString(args) - sig := md5Str(policy + "&" + uf.Secret) - - fd, err := os.Open(fpath) - if err != nil { - return nil, err - } - - defer fd.Close() - - url := fmt.Sprintf("http://%s/%s", uf.endpoint, uf.Bucket) - resp, err := uf.doFormRequest(url, policy, sig, fpath, fd) - if err != nil { - return nil, err - } - - defer resp.Body.Close() - - buf, err := ioutil.ReadAll(resp.Body) - if resp.StatusCode/100 == 2 { - var formResp FormAPIResp - if err := json.Unmarshal(buf, &formResp); err != nil { - return nil, err - } - return &formResp, nil - } - - return nil, errors.New(string(buf)) -} diff --git a/upyun/upyun-http-core.go b/upyun/upyun-http-core.go deleted file mode 100644 index bb6a34d..0000000 --- a/upyun/upyun-http-core.go +++ /dev/null @@ -1,102 +0,0 @@ -package upyun - -import ( - "bytes" - "io" - "mime/multipart" - "net" - "net/http" - "path/filepath" - "strconv" - "time" -) - -// Auto: Auto detected, based on user's internet -// Telecom: (ISP) China Telecom -// Cnc: (ISP) China Unicom -// Ctt: (ISP) China Tietong -const ( - Auto = iota - Telecom - Cnc - Ctt -) - -type upYunHTTPCore struct { - endpoint string - httpClient *http.Client -} - -func (core *upYunHTTPCore) SetTimeout(timeout time.Duration) { - core.httpClient = &http.Client{ - Transport: &http.Transport{ - Dial: func(network, addr string) (c net.Conn, err error) { - c, err = net.DialTimeout(network, addr, timeout) - if err != nil { - return nil, err - } - return - }, - // http://studygolang.com/articles/3138 - // DisableKeepAlives: true, - }, - } -} - -// do http form request -func (core *upYunHTTPCore) doFormRequest(url, policy, sign, - fname string, fd io.Reader) (*http.Response, error) { - - body := &bytes.Buffer{} - headers := make(map[string]string) - - // generate form data - err := func() error { - writer := multipart.NewWriter(body) - defer writer.Close() - - var err error - var part io.Writer - - writer.WriteField("policy", policy) - writer.WriteField("signature", sign) - if part, err = writer.CreateFormFile("file", filepath.Base(fname)); err == nil { - if _, err = chunkedCopy(part, fd); err == nil { - headers["Content-Type"] = writer.FormDataContentType() - } - } - return err - }() - - if err != nil { - return nil, err - } - - return core.doHTTPRequest("POST", url, headers, body) -} - -// do http request -func (core *upYunHTTPCore) doHTTPRequest(method, url string, headers map[string]string, - body io.Reader) (resp *http.Response, err error) { - req, err := http.NewRequest(method, url, body) - if err != nil { - return nil, err - } - - for k, v := range headers { - req.Header.Set(k, v) - } - - // User Agent - req.Header.Set("User-Agent", makeUserAgent()) - - // https://code.google.com/p/go/issues/detail?id=6738 - if method == "PUT" || method == "POST" { - length := req.Header.Get("Content-Length") - if length != "" { - req.ContentLength, _ = strconv.ParseInt(length, 10, 64) - } - } - - return core.httpClient.Do(req) -} diff --git a/upyun/upyun-media-api.go b/upyun/upyun-media-api.go deleted file mode 100644 index 4e2dc03..0000000 --- a/upyun/upyun-media-api.go +++ /dev/null @@ -1,166 +0,0 @@ -package upyun - -import ( - "encoding/json" - "errors" - "fmt" - "io/ioutil" - "net/http" - "sort" - "strings" -) - -// UPYUN MEDIA API -type UpYunMedia struct { - upYunHTTPCore // HTTP Core - - Username string - Passwd string - Bucket string -} - -// status response -type MediaStatusResp struct { - Tasks map[string]interface{} `json:"tasks"` -} - -// NewUpYunMedia returns a new UPYUN Media API client given -// a bucket name, username, password. http client connection -// timeout is set to defalutConnectionTimeout which -// is equal to 60 seconds. - -func NewUpYunMedia(bucket, user, pass string) *UpYunMedia { - up := &UpYunMedia{ - Username: user, - Passwd: md5Str(pass), - Bucket: bucket, - } - - client := &http.Client{} - up.SetTimeout(defaultConnectTimeout) - - up.endpoint = "p0.api.upyun.com" - up.httpClient = client - - return up -} - -func (upm *UpYunMedia) makeMediaAuth(kwargs map[string]string) string { - var keys []string - for k, _ := range kwargs { - keys = append(keys, k) - } - - sort.Strings(keys) - - auth := "" - for _, k := range keys { - auth += k + kwargs[k] - } - - return fmt.Sprintf("UPYUN %s:%s", upm.Username, - md5Str(upm.Username+auth+upm.Passwd)) -} - -// Send Media Tasks Reqeust -func (upm *UpYunMedia) PostTasks(src, notify, accept string, - tasks []map[string]interface{}) ([]string, error) { - data, err := json.Marshal(tasks) - if err != nil { - return nil, err - } - - kwargs := map[string]string{ - "bucket_name": upm.Bucket, - "source": src, - "notify_url": notify, - "tasks": base64Str(data), - "accept": accept, - } - - resp, err := upm.doMediaRequest("POST", "/pretreatment", kwargs) - if err != nil { - return nil, err - } - - defer resp.Body.Close() - - buf, err := ioutil.ReadAll(resp.Body) - if err != nil { - return nil, err - } - - if resp.StatusCode/2 == 100 { - var ids []string - err = json.Unmarshal(buf, &ids) - if err != nil { - return nil, err - } - return ids, err - } - - return nil, errors.New(string(buf)) -} - -// Get Task Progress -func (upm *UpYunMedia) GetProgress(task_ids string) (*MediaStatusResp, error) { - - kwargs := map[string]string{ - "bucket_name": upm.Bucket, - "task_ids": task_ids, - } - - resp, err := upm.doMediaRequest("GET", "/status", kwargs) - if err != nil { - return nil, err - } - - defer resp.Body.Close() - - buf, err := ioutil.ReadAll(resp.Body) - if resp.StatusCode/2 == 100 { - var status MediaStatusResp - if err := json.Unmarshal(buf, &status); err != nil { - return nil, err - } - return &status, nil - } - - return nil, errors.New(string(buf)) -} - -func (upm *UpYunMedia) doMediaRequest(method, path string, - kwargs map[string]string) (*http.Response, error) { - - // Normalize url - if !strings.HasPrefix(path, "/") { - path = "/" + path - } - url := fmt.Sprintf("http://%s%s", upm.endpoint, path) - - // Set Headers - headers := make(map[string]string) - date := genRFC1123Date() - headers["Date"] = date - headers["Authorization"] = upm.makeMediaAuth(kwargs) - - // Payload - var options []string - for k, v := range kwargs { - options = append(options, k+"="+v) - } - payload := strings.Join(options, "&") - - if method == "GET" { - url = url + "?" + payload - return upm.doHTTPRequest(method, url, headers, nil) - } else { - if method == "POST" { - headers["Content-Length"] = fmt.Sprint(len(payload)) - return upm.doHTTPRequest(method, url, headers, - strings.NewReader(payload)) - } - } - - return nil, errors.New("Unknown method") -} diff --git a/upyun/upyun-multipart-api.go b/upyun/upyun-multipart-api.go deleted file mode 100644 index dd0d72e..0000000 --- a/upyun/upyun-multipart-api.go +++ /dev/null @@ -1,301 +0,0 @@ -package upyun - -import ( - "bytes" - "encoding/base64" - "encoding/json" - "errors" - "fmt" - "io/ioutil" - "net/http" - "os" - "sort" - "strings" - "time" -) - -// UPYUN MultiPart Upload API -type UpYunMultiPart struct { - upYunHTTPCore - - Bucket string - Secret string - BlockSize int64 -} - -// upload response body -type UploadResp struct { - // returns after init request - SaveToken string `json:"save_token"` - // token_secert is equal to UPYUN Form API Secret - Secret string `json:"token_secret"` - // UPYUN Bucket Name - Bucket string `json:"bucket_name"` - // Number of Blocks - Blocks int `json:"blocks"` - Status []int `json:"status"` - ExpireAt int64 `json:"expire_at"` -} - -// merge response body -type MergeResp struct { - Path string `json:"path"` - ContentType string `json:"mimetype"` - ContentLength interface{} `json:"file_size"` - LastModify int64 `json:"last_modified"` - Signature string `json:"signature"` - ImageWidth int `json:"image_width"` - ImageHeight int `json:"image_height"` - ImageFrames int `json:"image_frames"` -} - -// NewUpYunMultiPart returns a new UPYUN Multipart Upload API client -// given bucket name, form api key and blocksize. -func NewUpYunMultiPart(bucket, secret string, blocksize int64) *UpYunMultiPart { - up := &UpYunMultiPart{ - Secret: secret, - Bucket: bucket, - BlockSize: blocksize, - } - - up.endpoint = "m0.api.upyun.com" - up.httpClient = &http.Client{ - Transport: &http.Transport{ - Dial: timeoutDialer(defaultConnectTimeout), - }, - } - - return up -} - -// make multipart upload authorization -func (ump *UpYunMultiPart) makeMPAuth(secret string, kwargs map[string]interface{}) string { - var keys []string - for k, _ := range kwargs { - keys = append(keys, k) - } - sort.Strings(keys) - - sign := "" - for _, k := range keys { - sign += k + fmt.Sprint(kwargs[k]) - } - - return md5Str(sign + secret) -} - -func (ump *UpYunMultiPart) makePolicy(kwargs map[string]interface{}) (string, error) { - data, err := json.Marshal(kwargs) - if err != nil { - return "", err - } - - return base64.StdEncoding.EncodeToString(data), nil -} - -// InitUpload initalizes a multipart upload request -func (ump *UpYunMultiPart) InitUpload(key string, value *os.File, - expire int64, options map[string]interface{}) ([]byte, error) { - // seek at start point - value.Seek(0, 0) - hash, fsize, err := md5sum(value) - if err != nil { - return nil, err - } - - opt := map[string]interface{}{ - "path": key, - "expiration": time.Now().UTC().Unix() + expire, - "file_hash": string(hash), - "file_size": fsize, - "file_blocks": (fsize + ump.BlockSize - 1) / ump.BlockSize, - } - if options != nil { - for k, v := range options { - opt[k] = v - } - } - - // make policy - policy, err := ump.makePolicy(opt) - if err != nil { - return nil, err - } - - // make signature - signature := ump.makeMPAuth(ump.Secret, opt) - payload := fmt.Sprintf("policy=%s&signature=%s", policy, signature) - - // set http headers - headers := map[string]string{ - "Content-Length": fmt.Sprint(len(payload)), - "Content-Type": "application/x-www-form-urlencoded", - } - - url := fmt.Sprintf("http://%s/%s/", ump.endpoint, ump.Bucket) - resp, err := ump.doHTTPRequest("POST", - url, headers, strings.NewReader(payload)) - - if err != nil { - return nil, err - } - - defer resp.Body.Close() - - body, err := ioutil.ReadAll(resp.Body) - if resp.StatusCode/100 == 2 { - return body, err - } - - return nil, errors.New(string(body)) -} - -// UploadBlock uploads a block -func (ump *UpYunMultiPart) UploadBlock(fd *os.File, bindex int, expire int64, - fpath, saveToken, secret string) ([]byte, error) { - - block := make([]byte, ump.BlockSize) - // seek to this block's start point - _, err := fd.Seek(ump.BlockSize*int64(bindex), 0) - if err != nil { - return nil, err - } - - // read block - n, err := fd.Read(block) - if err != nil { - return nil, err - } - rblock := block[:n] - - // calculate md5 - hash, _, err := md5sum(bytes.NewBuffer(rblock)) - if err != nil { - return nil, err - } - - opts := map[string]interface{}{ - "save_token": saveToken, - "expiration": time.Now().UTC().Unix() + expire, - "block_index": bindex, - "block_hash": string(hash), - } - - policy, err := ump.makePolicy(opts) - if err != nil { - return nil, err - } - - signature := ump.makeMPAuth(secret, opts) - url := fmt.Sprintf("http://%s/%s/", ump.endpoint, ump.Bucket) - - resp, err := ump.doFormRequest(url, policy, signature, fpath, bytes.NewBuffer(rblock)) - if err != nil { - return nil, err - } - - defer resp.Body.Close() - - body, err := ioutil.ReadAll(resp.Body) - if resp.StatusCode/100 == 2 { - return body, err - } - - return nil, errors.New(string(body)) -} - -// MergeBlock posts a merge request to merge all blocks uploaded -func (ump *UpYunMultiPart) MergeBlock(saveToken, secret string, - expire int64) ([]byte, error) { - opts := map[string]interface{}{ - "save_token": saveToken, - "expiration": time.Now().UTC().Unix() + expire, - } - - policy, err := ump.makePolicy(opts) - if err != nil { - return nil, err - } - - signature := ump.makeMPAuth(secret, opts) - payload := fmt.Sprintf("policy=%s&signature=%s", policy, signature) - - headers := map[string]string{ - "Content-Length": fmt.Sprint(len(payload)), - "Content-Type": "application/x-www-form-urlencoded", - } - - url := fmt.Sprintf("http://%s/%s/", ump.endpoint, ump.Bucket) - resp, err := ump.doHTTPRequest("POST", - url, headers, strings.NewReader(payload)) - - if err != nil { - return nil, err - } - - defer resp.Body.Close() - - body, err := ioutil.ReadAll(resp.Body) - if resp.StatusCode/100 == 2 { - return body, err - } - - return nil, errors.New(string(body)) -} - -// Put uploads a file through UPYUN MultiPart Upload API -func (ump *UpYunMultiPart) Put(fpath, saveas string, - expireAfter int64, options map[string]interface{}) (*MergeResp, error) { - fd, err := os.Open(fpath) - if err != nil { - return nil, err - } - defer fd.Close() - - rdata, err := ump.InitUpload(saveas, fd, expireAfter, options) - if err != nil { - return nil, errors.New("failed to init upload." + err.Error()) - } - - var ub UploadResp - if err := json.Unmarshal(rdata, &ub); err != nil { - return nil, err - } - - saveToken := ub.SaveToken - secret := ub.Secret - status := ub.Status - for try := 1; try <= 3; try++ { - ok := 0 - for idx, _ := range status { - if status[idx] == 0 { - _, err = ump.UploadBlock(fd, idx, expireAfter, fpath, saveToken, secret) - if err != nil { - break - } - status[idx] = 1 - } - ok++ - } - - if ok == len(status) { - break - } - - if try == 3 { - return nil, errors.New("failed to upload block." + err.Error()) - } - } - - data, err := ump.MergeBlock(saveToken, secret, expireAfter) - if err != nil { - return nil, errors.New("failed to merge blocks." + err.Error()) - } - - var mr MergeResp - if err := json.Unmarshal(data, &mr); err != nil { - return nil, err - } - - return &mr, nil -} diff --git a/upyun/upyun-rest-api.go b/upyun/upyun-rest-api.go deleted file mode 100644 index 3bce0c6..0000000 --- a/upyun/upyun-rest-api.go +++ /dev/null @@ -1,495 +0,0 @@ -package upyun - -import ( - "bytes" - "crypto/md5" - "encoding/json" - "errors" - "fmt" - "io" - "io/ioutil" - "net" - "net/http" - URL "net/url" - "os" - "path" - "strconv" - "strings" - "time" -) - -// UPYUN REST API Client -type UpYun struct { - // Core - upYunHTTPCore - - Bucket string - Username string - Passwd string - ChunkSize int -} - -// NewUpYun return a new UPYUN REST API client given a bucket name, -// username, password. As Default, endpoint is set to Auto, http -// client connection timeout is set to defalutConnectionTimeout which -// is equal to 60 seconds. -func NewUpYun(bucket, username, passwd string) *UpYun { - u := &UpYun{ - Bucket: bucket, - Username: username, - Passwd: passwd, - } - - u.httpClient = &http.Client{} - u.SetEndpoint(Auto) - u.SetTimeout(defaultConnectTimeout) - - return u -} - -// SetEndpoint sets the request endpoint to UPYUN REST API Server. -func (u *UpYun) SetEndpoint(ed int) error { - if ed >= Auto && ed <= Ctt { - u.endpoint = fmt.Sprintf("v%d.api.upyun.com", ed) - return nil - } - - return errors.New("Invalid endpoint, pick from Auto, Telecom, Cnc, Ctt") -} - -// SetEndpointStr sets the request endpoint to UPYUN REST API Server. -func (u *UpYun) SetEndpointStr(endpoint string) error { - u.endpoint = endpoint - return nil -} - -// make UpYun REST Authorization -func (u *UpYun) makeRESTAuth(method, uri, date, lengthStr string) string { - sign := []string{method, uri, date, lengthStr, md5Str(u.Passwd)} - - return "UpYun " + u.Username + ":" + md5Str(strings.Join(sign, "&")) -} - -// make UpYun Purge Authorization -func (u *UpYun) makePurgeAuth(purgeList, date string) string { - sign := []string{purgeList, u.Bucket, date, md5Str(u.Passwd)} - - return "UpYun " + u.Bucket + ":" + u.Username + ":" + md5Str(strings.Join(sign, "&")) -} - -// Usage gets the usage of the bucket in UPYUN File System -func (u *UpYun) Usage() (int64, error) { - result, _, err := u.doRESTRequest("GET", "/", "usage", nil, nil) - if err != nil { - return 0, err - } - - return strconv.ParseInt(result, 10, 64) -} - -// Mkdir creates a directory in UPYUN File System -func (u *UpYun) Mkdir(key string) error { - headers := make(map[string]string) - - headers["mkdir"] = "true" - headers["folder"] = "true" - - _, _, err := u.doRESTRequest("POST", key, "", headers, nil) - - return err -} - -// Put uploads filelike object to UPYUN File System -func (u *UpYun) Put(key string, value io.Reader, useMD5 bool, - headers map[string]string) (http.Header, error) { - if headers == nil { - headers = make(map[string]string) - } - - if _, ok := headers["Content-Length"]; !ok { - switch v := value.(type) { - case *os.File: - if fileInfo, err := v.Stat(); err != nil { - return nil, err - } else { - headers["Content-Length"] = fmt.Sprint(fileInfo.Size()) - } - default: - // max buffer is 10k - rw := bytes.NewBuffer(make([]byte, 0, 10240)) - if n, err := io.Copy(rw, value); err != nil { - return nil, err - } else { - headers["Content-Length"] = fmt.Sprint(n) - } - value = rw - } - } - - if _, ok := headers["Content-MD5"]; !ok && useMD5 { - switch v := value.(type) { - case *os.File: - hash := md5.New() - if _, err := io.Copy(hash, value); err != nil { - return nil, err - } - - headers["Content-MD5"] = fmt.Sprintf("%x", hash.Sum(nil)) - - if _, err := v.Seek(0, 0); err != nil { - return nil, err - } - } - } - - _, rtHeaders, err := u.doRESTRequest("PUT", key, "", headers, value) - - return rtHeaders, err -} - -// Put uploads file object to UPYUN File System part by part, -// and automatically retries when a network problem occurs -func (u *UpYun) ResumePut(key string, value *os.File, useMD5 bool, - headers map[string]string, reporter ResumeReporter) (http.Header, error) { - if headers == nil { - headers = make(map[string]string) - } - - fileinfo, err := value.Stat() - if err != nil { - return nil, err - } - - // If filesize < resumePartSizeLowerLimit, use UpYun.Put() instead - if fileinfo.Size() < resumeFileSizeLowerLimit { - return u.Put(key, value, useMD5, headers) - } - - maxPartID := int(fileinfo.Size() / resumePartSize) - if fileinfo.Size()%resumePartSize == 0 { - maxPartID-- - } - - var resp http.Header - - for part := 0; part <= maxPartID; part++ { - - innerHeaders := make(map[string]string) - for k, v := range headers { - innerHeaders[k] = v - } - - innerHeaders["X-Upyun-Part-Id"] = strconv.Itoa(part) - switch part { - case 0: - innerHeaders["X-Upyun-Multi-Type"] = headers["Content-Type"] - innerHeaders["X-Upyun-Multi-Length"] = strconv.FormatInt(fileinfo.Size(), 10) - innerHeaders["X-Upyun-Multi-Stage"] = "initiate,upload" - innerHeaders["Content-Length"] = strconv.Itoa(resumePartSize) - case maxPartID: - innerHeaders["X-Upyun-Multi-Stage"] = "upload,complete" - innerHeaders["Content-Length"] = fmt.Sprint(fileinfo.Size() - int64(resumePartSize)*int64(part)) - if useMD5 { - value.Seek(0, 0) - hex, _, _ := md5sum(value) - innerHeaders["X-Upyun-Multi-MD5"] = hex - } - default: - innerHeaders["X-Upyun-Multi-Stage"] = "upload" - innerHeaders["Content-Length"] = strconv.Itoa(resumePartSize) - } - - file, err := NewFragmentFile(value, int64(part)*int64(resumePartSize), resumePartSize) - if err != nil { - return resp, err - } - if useMD5 { - innerHeaders["Content-MD5"], _ = file.MD5() - } - - // Retry when get net error from UpYun.Put(), return error in other cases - for i := 0; i < ResumeRetryCount+1; i++ { - resp, err = u.Put(key, file, useMD5, innerHeaders) - if err == nil { - break - } - // Retry only get net error - _, ok := err.(net.Error) - if !ok { - return resp, err - } - if i == ResumeRetryCount { - return resp, err - } - time.Sleep(ResumeWaitTime) - file.Seek(0, 0) - } - if reporter != nil { - reporter(part, maxPartID) - } - - if part == 0 { - headers["X-Upyun-Multi-UUID"] = resp.Get("X-Upyun-Multi-Uuid") - } - } - - return resp, nil -} - -// Get gets the specified file in UPYUN File System -func (u *UpYun) Get(key string, value io.Writer) (int, error) { - length, _, err := u.doRESTRequest("GET", key, "", nil, value) - if err != nil { - return 0, err - } - return strconv.Atoi(length) -} - -// Delete deletes the specified **file** in UPYUN File System. -func (u *UpYun) Delete(key string) error { - _, _, err := u.doRESTRequest("DELETE", key, "", nil, nil) - - return err -} - -// AsyncDelete deletes the specified **file** in UPYUN File System asynchronously. -func (u *UpYun) AsyncDelete(key string) error { - headers := map[string]string{ - "X-Upyun-Async": "true", - } - _, _, err := u.doRESTRequest("DELETE", key, "", headers, nil) - - return err -} - -// GetList lists items in key. The number of items must be -// less then 100 -func (u *UpYun) GetList(key string) ([]*FileInfo, error) { - ret, _, err := u.doRESTRequest("GET", key, "", nil, nil) - if err != nil { - return nil, err - } - - list := strings.Split(ret, "\n") - var infoList []*FileInfo - - for _, v := range list { - if len(v) == 0 { - continue - } - infoList = append(infoList, newFileInfo(v)) - } - - return infoList, nil -} - -// Note: key must be directory -func (u *UpYun) GetLargeList(key string, asc, recursive bool) (chan *FileInfo, chan error) { - infoChannel := make(chan *FileInfo, 1000) - errChannel := make(chan error, 10) - if !strings.HasSuffix(key, "/") { - key += "/" - } - order := "desc" - if asc == true { - order = "asc" - } - - go func() { - var listDir func(k string) error - listDir = func(k string) error { - var infos []*FileInfo - var niter string - var err error - iter, limit := "", 50 - for { - infos, niter, err = u.loopList(k, iter, order, limit) - if err != nil { - errChannel <- err - return err - } - iter = niter - for _, f := range infos { - // absolute path - abs := path.Join(k, f.Name) - // relative path - f.Name = strings.Replace(abs, key, "", 1) - if f.Name[0] == '/' { - f.Name = f.Name[1:] - } - if recursive && f.Type == "folder" { - if err = listDir(abs + "/"); err != nil { - return err - } - } - infoChannel <- f - } - if iter == "" { - break - } - } - return nil - } - - listDir(key) - - close(errChannel) - close(infoChannel) - }() - - return infoChannel, errChannel -} - -// LoopList list items iteratively. -func (u *UpYun) loopList(key, iter, order string, limit int) ([]*FileInfo, string, error) { - headers := map[string]string{ - "X-List-Limit": fmt.Sprint(limit), - "X-List-Order": order, - } - if iter != "" { - headers["X-List-Iter"] = iter - } - - ret, rtHeaders, err := u.doRESTRequest("GET", key, "", headers, nil) - if err != nil { - return nil, "", err - } - - list := strings.Split(ret, "\n") - var infoList []*FileInfo - for _, v := range list { - if len(v) == 0 { - continue - } - infoList = append(infoList, newFileInfo(v)) - } - - nextIter := "" - if _, ok := rtHeaders["X-Upyun-List-Iter"]; ok { - nextIter = rtHeaders["X-Upyun-List-Iter"][0] - } else { - // Maybe Wrong - return nil, "", nil - } - - if nextIter == "g2gCZAAEbmV4dGQAA2VvZg" { - nextIter = "" - } - - return infoList, nextIter, nil -} - -// GetInfo gets information of item in UPYUN File System -func (u *UpYun) GetInfo(key string) (*FileInfo, error) { - _, headers, err := u.doRESTRequest("HEAD", key, "", nil, nil) - if err != nil { - return nil, err - } - - fileInfo := newFileInfo(headers) - - return fileInfo, nil -} - -// Purge post a purge request to UPYUN Purge Server -func (u *UpYun) Purge(urls []string) (string, error) { - purge := "http://purge.upyun.com/purge/" - - date := genRFC1123Date() - purgeList := strings.Join(urls, "\n") - - headers := make(map[string]string) - headers["Date"] = date - headers["Authorization"] = u.makePurgeAuth(purgeList, date) - headers["Content-Type"] = "application/x-www-form-urlencoded;charset=utf-8" - - form := make(URL.Values) - form.Add("purge", purgeList) - - body := strings.NewReader(form.Encode()) - resp, err := u.doHTTPRequest("POST", purge, headers, body) - defer resp.Body.Close() - - content, err := ioutil.ReadAll(resp.Body) - if err != nil { - return "", err - } - - if resp.StatusCode/100 == 2 { - result := make(map[string][]string) - if err := json.Unmarshal(content, &result); err != nil { - // quick fix for invalid json resp: {"invalid_domain_of_url":{}} - return "", nil - } - - return strings.Join(result["invalid_domain_of_url"], ","), nil - } - - return "", errors.New(string(content)) -} - -func (u *UpYun) doRESTRequest(method, uri, query string, headers map[string]string, - value interface{}) (result string, rtHeaders http.Header, err error) { - if headers == nil { - headers = make(map[string]string) - } - - // Normalize url - if !strings.HasPrefix(uri, "/") { - uri = "/" + uri - } - - uri = escapeURI("/" + u.Bucket + uri) - url := fmt.Sprintf("http://%s%s", u.endpoint, uri) - - if query != "" { - query = escapeURI(query) - url += "?" + query - } - - // date - date := genRFC1123Date() - - // auth - lengthStr, ok := headers["Content-Length"] - if !ok { - lengthStr = "0" - } - - headers["Date"] = date - headers["Authorization"] = u.makeRESTAuth(method, uri, date, lengthStr) - if !strings.Contains(u.endpoint, "api.upyun.com") { - headers["Host"] = "v0.api.upyun.com" - } - - // HEAD GET request has no body - rc, ok := value.(io.Reader) - if !ok || method == "GET" || method == "HEAD" { - rc = nil - } - - resp, err := u.doHTTPRequest(method, url, headers, rc) - if err != nil { - return "", nil, err - } - - defer resp.Body.Close() - - if (resp.StatusCode / 100) == 2 { - if method == "GET" && value != nil { - written, err := chunkedCopy(value.(io.Writer), resp.Body) - return strconv.FormatInt(written, 10), resp.Header, err - } - body, err := ioutil.ReadAll(resp.Body) - return string(body), resp.Header, err - } - - if body, err := ioutil.ReadAll(resp.Body); err == nil { - if len(body) == 0 && resp.StatusCode/100 != 2 { - return "", resp.Header, errors.New(fmt.Sprint(resp.StatusCode)) - } - return "", resp.Header, errors.New(string(body)) - } else { - return "", resp.Header, err - } -} diff --git a/upyun/upyun-resume.go b/upyun/upyun-resume.go deleted file mode 100644 index 42fae87..0000000 --- a/upyun/upyun-resume.go +++ /dev/null @@ -1,93 +0,0 @@ -package upyun - -import ( - "errors" - "fmt" - "io" - "os" - "time" -) - -const ( - // resumePartSize is the size of each part for resume upload - resumePartSize = 1024 * 1024 - // resumeFileSizeLowerLimit is the lowest file size limit for resume upload - resumeFileSizeLowerLimit int64 = resumePartSize * 10 -) - -var ( - // ResumeRetry is the number of retries for resume upload - ResumeRetryCount = 3 - // ResumeWaitSeconds is the number of time to wait when net error occurs - ResumeWaitTime = time.Second * 5 -) - -// ResumeReporter -type ResumeReporter func(int, int) - -// ResumeReporterPrintln is a simple ResumeReporter for test -func ResumeReporterPrintln(partID int, maxPartID int) { - fmt.Printf("resume test reporter: %v / %v\n", partID, maxPartID) -} - -// FragmentFile is like os.File, but only a part of file can be Read(). -// return io.EOF when cursor fetch the limit. -type FragmentFile struct { - offset int64 - limit int - cursor int - *os.File -} - -// NewFragmentFile returns a new FragmentFile. -func NewFragmentFile(file *os.File, offset int64, limit int) (*FragmentFile, error) { - sizedfile := &FragmentFile{ - offset: offset, - limit: limit, - File: file, - } - _, err := sizedfile.Seek(0, 0) - if err != nil { - return nil, err - } - return sizedfile, nil -} - -// Seek likes os.File.Seek() -func (f *FragmentFile) Seek(offset int64, whence int) (ret int64, err error) { - switch whence { - case 0: - f.cursor = int(offset) - return f.File.Seek(f.offset+offset, 0) - default: - return 0, errors.New("whence must be 0") - } -} - -// Read is just like os.File.Read but return io.EOF when catch sizedfile's limit -// or the end of file -func (f *FragmentFile) Read(b []byte) (n int, err error) { - if f.cursor >= f.limit { - return 0, io.EOF - } - n, err = f.File.Read(b) - if int(f.cursor)+n > f.limit { - n = f.limit - f.cursor - } - f.cursor += n - return n, err -} - -// Close will not actually close FragmentFile -func (f *FragmentFile) Close() error { - return nil -} - -// MD5 returns md5 of the FragmentFile. -func (f *FragmentFile) MD5() (string, error) { - cursor := f.cursor - f.Seek(0, 0) - md5, _, err := md5sum(f) - f.Seek(int64(cursor), 0) - return md5, err -} diff --git a/upyun/upyun.go b/upyun/upyun.go index 8dfc5e2..a8889f7 100644 --- a/upyun/upyun.go +++ b/upyun/upyun.go @@ -1,174 +1,61 @@ -// package upyun is used for your UPYUN bucket -// this sdk implement purge api, form api, http rest api package upyun import ( - "crypto/md5" - "encoding/base64" - "encoding/hex" - "fmt" - "io" "net" "net/http" - URL "net/url" - "strconv" - "strings" "time" ) const ( - Version = "2.0.0" -) + version = "0.1.0" -const ( - // Default(Min/Max)ChunkSize: set the buffer size when doing copy operation - defaultChunkSize = 32 * 1024 - // defaultConnectTimeout: connection timeout when connect to upyun endpoint + defaultChunkSize = 32 * 1024 defaultConnectTimeout = time.Second * 60 ) -// chunkSize: chunk size when copy -var ( - chunkSize = defaultChunkSize -) - -// Util functions - -// User Agent -func makeUserAgent() string { - return fmt.Sprintf("UPYUN Go SDK %s", Version) -} - -// Greenwich Mean Time -func genRFC1123Date() string { - return time.Now().UTC().Format(time.RFC1123) -} - -// make md5 from string -func md5Str(s string) (ret string) { - return fmt.Sprintf("%x", md5.Sum([]byte(s))) -} - -// make base64 from []byte -func base64Str(b []byte) string { - return base64.StdEncoding.EncodeToString(b) -} - -// URL encode -func encodeURL(uri string) string { - return base64.URLEncoding.EncodeToString([]byte(uri)) -} - -// URI escape -func escapeURI(uri string) string { - Uri := URL.URL{} - Uri.Path = uri - return Uri.String() -} - -func md5sum(fd io.Reader) (string, int64, error) { - var result []byte - hash := md5.New() - if written, err := io.Copy(hash, fd); err != nil { - return "", written, err +type UpYunConfig struct { + Bucket string + Operator string + Password string + Secret string // deprecated + Hosts map[string]string + UserAgent string +} + +type UpYun struct { + UpYunConfig + httpc *http.Client + deprecated bool +} + +func NewUpYun(config *UpYunConfig) *UpYun { + up := &UpYun{} + up.Bucket = config.Bucket + up.Operator = config.Operator + up.Password = md5Str(config.Password) + up.Secret = config.Secret + up.Hosts = config.Hosts + if config.UserAgent != "" { + up.UserAgent = config.UserAgent } else { - return hex.EncodeToString(hash.Sum(result)), written, nil - } -} - -// Because of io.Copy use a 32Kb buffer, and, it is hard coded -// user can specify a chunksize with upyun.SetChunkSize -func chunkedCopy(dst io.Writer, src io.Reader) (written int64, err error) { - buf := make([]byte, chunkSize) - - for { - nr, er := src.Read(buf) - if nr > 0 { - nw, ew := dst.Write(buf[0:nr]) - - if nw > 0 { - written += int64(nw) - } - if ew != nil { - err = ew - break - } - if nr != nw { - err = io.ErrShortWrite - break - } - } - if er == io.EOF { - break - } - if er != nil { - err = er - break - } + up.UserAgent = makeUserAgent(version) } - return -} -// Use for http connection timeout -func timeoutDialer(timeout time.Duration) func(string, string) (net.Conn, error) { - return func(network, addr string) (c net.Conn, err error) { - c, err = net.DialTimeout(network, addr, timeout) - if err != nil { - return nil, err - } - return + up.httpc = &http.Client{ + Transport: &http.Transport{ + Dial: func(network, addr string) (c net.Conn, err error) { + return net.DialTimeout(network, addr, defaultConnectTimeout) + }, + }, } -} -func SetChunkSize(chunksize int) { - chunkSize = chunksize + return up } -// FileInfo when use getlist -type FileInfo struct { - Size int64 - Time time.Time - Name string - Type string +func (up *UpYun) SetHTTPClient(httpc *http.Client) { + up.httpc = httpc } -func newFileInfo(arg interface{}) *FileInfo { - switch arg.(type) { - case string: - s := arg.(string) - infoList := strings.Split(s, "\t") - if len(infoList) != 4 { - return nil - } - - size, _ := strconv.ParseInt(infoList[2], 10, 64) - timestamp, _ := strconv.ParseInt(infoList[3], 10, 64) - typ := "folder" - if infoList[1] != "F" { - typ = "file" - } - - return &FileInfo{ - Name: infoList[0], - Type: typ, - Size: size, - Time: time.Unix(timestamp, 0), - } - - default: - var fileInfo FileInfo - headers := arg.(http.Header) - for k, v := range headers { - switch { - case strings.Contains(k, "File-Type"): - fileInfo.Type = v[0] - case strings.Contains(k, "File-Size"): - fileInfo.Size, _ = strconv.ParseInt(v[0], 10, 64) - case strings.Contains(k, "File-Date"): - timestamp, _ := strconv.ParseInt(v[0], 10, 64) - fileInfo.Time = time.Unix(timestamp, 0) - } - } - return &fileInfo - } +func (up *UpYun) UseDeprecatedApi() { + up.deprecated = true } diff --git a/upyun/upyun_test.go b/upyun/upyun_test.go index a347aed..d90e36e 100644 --- a/upyun/upyun_test.go +++ b/upyun/upyun_test.go @@ -1,346 +1,123 @@ package upyun import ( - "bytes" - "crypto/md5" + "flag" "fmt" - "io" "os" - "strings" + "path" + "path/filepath" + "reflect" + "runtime" + "sync" "testing" + "time" ) var ( - username = os.Getenv("UPYUN_USERNAME") - password = os.Getenv("UPYUN_PASSWORD") - bucket = os.Getenv("UPYUN_BUCKET") - apikey = os.Getenv("UPYUN_SECRET") - up = NewUpYun(bucket, username, password) - upf = NewUpYunForm(bucket, apikey) - ump = NewUpYunMultiPart(bucket, apikey, 1024000) - upm = NewUpYunMedia(bucket, username, password) - testPath = "/gosdk" - upload = "upyun-rest-api.go" - uploadInfo, _ = os.Lstat(upload) - uploadSize = uploadInfo.Size() - download = "/tmp/xxx.go" - - length int - err error - fd *os.File - upInfo *FileInfo - upInfos []*FileInfo - formResp *FormAPIResp - mergeResp *MergeResp + ROOT = MakeTmpPath() + NOTIFY_URL = os.Getenv("UPYUN_NOTIFY") ) -func TestUsage(t *testing.T) { - if _, err := up.Usage(); err != nil { - fmt.Println(err) - t.Errorf("failed to get Usage. %v", err) - } -} - -func TestSetEndpoint(t *testing.T) { - for _, ed := range []int{Telecom, Cnc, Ctt, Auto} { - if err = up.SetEndpoint(ed); err == nil { - _, err = up.Usage() - } - if err != nil { - t.Errorf("failed to SetEndpoint. %v %v", ed, err) - } - } - if err = up.SetEndpoint(5); err == nil { - t.Errorf("invalid SetEndpoint") - } -} - -func TestMkdir(t *testing.T) { - if _, err = up.GetInfo(testPath); err == nil { - t.Error(testPath, "already exists") - // t.Fail() - } - if err = up.Mkdir(testPath); err == nil { - _, err = up.GetInfo(testPath) - } - if err != nil { - t.Errorf("failed to Mkdir. %v", err) - } -} - -func TestPut(t *testing.T) { - // put file - if fd, err = os.Open(upload); err != nil { - t.Skipf("failed to open %s %v", upload, err) - } - - _, err = up.Put(testPath+"/"+upload, fd, false, nil) - if err != nil { - t.Errorf("failed to put %v", err) - } - - fd, _ = os.Open(upload) - _, err = up.Put(testPath+"/dir2/"+upload, fd, true, map[string]string{"Content-Type": "video/mp4"}) - if err != nil { - t.Errorf("failed to put %v", err) - } - - // put buf - b := bytes.NewReader([]byte("UPYUN GO SDK")) - _, err = up.Put(testPath+"/"+upload+".buf", b, false, nil) - if err != nil { - t.Errorf("failed to put %v", err) - } -} - -func TestGet(t *testing.T) { - fd, err = os.OpenFile(download, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0600) - if err != nil { - t.Skipf("failed to open %s %v", download, err) - } - - defer os.Remove(download) - - if length, err = up.Get(testPath+"/"+upload, fd); err != nil { - t.Errorf("failed to get %s %v", testPath+"/"+upload, err) - } - - if length != int(uploadSize) { - t.Errorf("size not equal %d != %d", length, uploadSize) - } +var up = NewUpYun(&UpYunConfig{ + Bucket: os.Getenv("UPYUN_BUCKET"), + Operator: os.Getenv("UPYUN_USERNAME"), + Password: os.Getenv("UPYUN_PASSWORD"), + Secret: os.Getenv("UPYUN_SECRET"), +}) - dInfo, _ := fd.Stat() - if dInfo.Size() != uploadSize { - t.Errorf("size not equal %d != %d", dInfo.Size(), uploadSize) - } +func MakeTmpPath() string { + return "/go-sdk/" + time.Now().String() } -func TestGetInfo(t *testing.T) { - if upInfo, err = up.GetInfo(testPath); err != nil { - t.Errorf("failed to GetInfo %s %v", testPath, err) - } - if upInfo.Type != "folder" { - t.Errorf("%s not folder", testPath) - } - - if upInfo, err = up.GetInfo(testPath + "/" + upload); err != nil { - t.Errorf("failed to GetInfo %s %v", testPath+"/"+upload, err) - } else { - if upInfo.Type != "file" { - t.Errorf("%s not file", testPath+"/"+upload) - } - if upInfo.Size != uploadSize { - t.Errorf("size not equal %d != %d", upInfo.Size, uploadSize) - } - } - - if upInfo, err = up.GetInfo(testPath + "/up"); upInfo != nil || err == nil { - t.Errorf("%s should not exist", testPath+"/up") - } -} - -func TestGetList(t *testing.T) { - if upInfos, err = up.GetList(testPath); err != nil { - t.Errorf("failed to GetList %s %v", testPath, err) - } - - if len(upInfos) != 3 { - t.Errorf("failed to GetList %s %d != 3", testPath, len(upInfos)) +func Equal(t *testing.T, actual, expected interface{}) { + if !reflect.DeepEqual(actual, expected) { + _, file, line, _ := runtime.Caller(1) + t.Logf("\033[31m%s:%d:\n\n\tnexp: %#v\n\n\tgot: %#v\033[39m\n\n", + filepath.Base(file), line, expected, actual) + t.FailNow() } } -func TestGetLargeList(t *testing.T) { - ch, _ := up.GetLargeList(testPath, false, false) - count := 0 - for { - var more bool - upInfo, more = <-ch - if !more { - break - } - count++ - } - if count != 3 { - t.Errorf("GetLargeList %d != 3", count) - } - - ch, _ = up.GetLargeList(testPath, true, true) - count = 0 - for { - var more bool - upInfo, more = <-ch - if !more { - break - } - count++ - } - if count != 4 { - t.Errorf("GetLargeList recursive %d != 4", count) +func NotEqual(t *testing.T, actual, expected interface{}) { + if reflect.DeepEqual(actual, expected) { + _, file, line, _ := runtime.Caller(1) + t.Logf("\033[31m%s:%d:\n\n\tnexp: %#v\n\n\tgot: %#v\033[39m\n\n", + filepath.Base(file), line, expected, actual) + t.FailNow() } } -func TestResumeSmallFile(t *testing.T) { - file, err := os.Open(upload) - if err != nil { - t.Error(err) - } - defer file.Close() - - md5Hash := md5.New() - io.Copy(md5Hash, file) - md5 := fmt.Sprintf("%x", md5Hash.Sum(nil)) - file.Seek(0, 0) - - _, err = up.ResumePut(testPath+"/"+upload, file, true, map[string]string{"Content-Type": "text/plain"}, nil) - if err != nil { - t.Error(err) - } - - _, err = up.GetInfo(testPath + "/" + upload) - if err != nil { - t.Error(err) - } - - buf := bytes.NewBuffer(make([]byte, 0, 1024)) - up.Get(testPath+"/"+upload, buf) - - md5Hash.Reset() - io.Copy(md5Hash, buf) - - if fmt.Sprintf("%x", md5Hash.Sum(nil)) != md5 { - t.Error("MD5 is inconsistent") +func Nil(t *testing.T, object interface{}) { + if !isNil(object) { + _, file, line, _ := runtime.Caller(1) + t.Logf("\033[31m%s:%d:\n\n\t (expected)\n\n\t!= %#v (actual)\033[39m\n\n", + filepath.Base(file), line, object) + t.FailNow() } } -func TestResumeBigFile(t *testing.T) { - file, err := os.OpenFile("/tmp/bigfile", os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0666) - if err != nil { - t.Error(err) - } - defer os.Remove(file.Name()) - defer file.Close() - - for i := 0; i < 15*1024; i++ { - file.Write(bytes.Repeat([]byte("1"), 1024)) - } - file.Seek(0, 0) - - md5Hash := md5.New() - io.Copy(md5Hash, file) - md5 := fmt.Sprintf("%x", md5Hash.Sum(nil)) - file.Seek(0, 0) - - _, err = up.ResumePut(testPath+"/"+"bigfile", file, true, nil, ResumeReporterPrintln) - if err != nil { - t.Error(err) - } - - defer up.Delete(testPath + "/" + "bigfile") - - _, err = up.GetInfo(testPath + "/" + "bigfile") - if err != nil { - t.Error(err) - } - - buf := bytes.NewBuffer(make([]byte, 0, 1024)) - up.Get(testPath+"/"+"bigfile", buf) - - md5Hash.Reset() - io.Copy(md5Hash, buf) - if fmt.Sprintf("%x", md5Hash.Sum(nil)) != md5 { - t.Error("MD5 is inconsistent") +func NotNil(t *testing.T, object interface{}) { + if isNil(object) { + _, file, line, _ := runtime.Caller(1) + t.Logf("\033[31m%s:%d:\n\n\tExpected value not to be \033[39m\n\n", + filepath.Base(file), line, object) + t.FailNow() } } -func TestDelete(t *testing.T) { - // delete file - path := testPath + "/" + upload - if err = up.Delete(path); err != nil { - t.Errorf("failed to Delete %s %v", path, err) - } - - path = testPath + "/" + upload + ".buf" - if err = up.Delete(path); err != nil { - t.Errorf("failed to Delete %s %v", path, err) - } - - path = testPath + "/dir2/" + upload - if err = up.Delete(path); err != nil { - t.Errorf("failed to Delete %s %v", path, err) +func isNil(object interface{}) bool { + if object == nil { + return true } - // delete not empty folder - path = testPath - if err = up.Delete(path); err == nil { - t.Errorf("Delete no-empty folder should failed %s", path) - } - // delete empty folder - path = testPath + "/dir2" - if err = up.Delete(path); err != nil { - t.Errorf("failed to Delete empty folder %s %v", path, err) + value := reflect.ValueOf(object) + kind := value.Kind() + if kind >= reflect.Chan && kind <= reflect.Slice && value.IsNil() { + return true } - path = testPath - if err = up.Delete(path); err != nil { - t.Errorf("failed to Delete empty folder %s %v", path, err) - } -} - -func TestPurge(t *testing.T) { - var s string - s, err = up.Purge([]string{"http://www.baidu.com", - fmt.Sprintf("http://%s.b0.upaiyun.com/%s", up.Bucket, testPath+"/"+upload)}) - if err != nil { - t.Errorf("failed to Purge %v", err) - } + return false - if s != "http://www.baidu.com" { - t.Errorf("%s != baidu", s) - } } -func TestFormAPI(t *testing.T) { - formResp, err = upf.Put(upload, - testPath+"/upload_{filename}{.suffix}", 3600, nil) - if err != nil { - t.Errorf("failed to put %s %v", upload, err) - return - } - if err = up.Delete(formResp.Url); err == nil { - err = up.Delete(testPath) - } - if err != nil { - t.Errorf("failed to remove %s %v", formResp.Url, err) - } -} - -func TestMultiPart(t *testing.T) { - mergeResp, err = ump.Put(upload, testPath+"/multipart", 3600, nil) - if err != nil { - t.Errorf("failed to put %s %v", upload, err) - return - } - - if err = up.Delete(mergeResp.Path); err == nil { - err = up.Delete(testPath) - } - if err != nil { - t.Errorf("failed to remove %s %v", mergeResp.Path, err) +func TestMain(m *testing.M) { + _, err := up.Usage() + if err != nil { + fmt.Println("failed to login. Have set UPYUN_BUCKET UPYUN_USERNAME UPYUN_PASSWORD UPYUN_SECRET?") + os.Exit(-1) + } + clean := func() { + objs := make(chan *FileInfo, 20) + var wg sync.WaitGroup + wg.Add(1) + go func() { + for obj := range objs { + up.Delete(&DeleteObjectConfig{ + Path: path.Join(ROOT, obj.Name), + }) + } + up.Delete(&DeleteObjectConfig{ + Path: ROOT, + }) + wg.Done() + }() + + up.List(&GetObjectsConfig{ + Path: ROOT, + ObjectsChan: objs, + MaxListLevel: -1, + }) + wg.Wait() + + if _, err := up.GetInfo(ROOT); err == nil { + fmt.Println("Not cleanup") + os.Exit(-1) + } } -} -func TestMedia(t *testing.T) { - task := map[string]interface{}{ - "type": "thumbnail", - "thumb_single": true, - } - tasks := []map[string]interface{}{task} + flag.Parse() + code := m.Run() - if ids, err := upm.PostTasks("kai.3gp", "http://www.upyun.com/notify", "json", tasks); err != nil { - t.Errorf("failed to post tasks %v %v", tasks, err) - } else { - if _, err = upm.GetProgress(strings.Join(ids, ",")); err != nil { - t.Errorf("failed to get progress %v %v", ids, err) - } - } + clean() + os.Exit(code) } diff --git a/upyun/utils.go b/upyun/utils.go new file mode 100644 index 0000000..f3308e3 --- /dev/null +++ b/upyun/utils.go @@ -0,0 +1,128 @@ +package upyun + +import ( + "crypto/hmac" + "crypto/md5" + "crypto/sha1" + "encoding/base64" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/url" + "path" + "strconv" + "strings" + "time" +) + +func makeRFC1123Date(d time.Time) string { + utc := d.UTC().Format(time.RFC1123) + return strings.Replace(utc, "UTC", "GMT", -1) +} + +func makeUserAgent(version string) string { + return fmt.Sprintf("UPYUN Go SDK V2/%s", version) +} + +func md5Str(s string) string { + return fmt.Sprintf("%x", md5.Sum([]byte(s))) +} + +func base64ToStr(b []byte) string { + return base64.StdEncoding.EncodeToString(b) +} + +func hmacSha1(key string, data []byte) []byte { + hm := hmac.New(sha1.New, []byte(key)) + hm.Write(data) + return hm.Sum(nil) +} + +func escapeUri(uri string) (string, error) { + uri = path.Join("/", uri) + u, err := url.ParseRequestURI(uri) + if err != nil { + return "", err + } + return u.String(), nil +} + +var readHTTPBody = ioutil.ReadAll + +func readHTTPBodyToStr(resp *http.Response) (string, error) { + b, err := readHTTPBody(resp.Body) + resp.Body.Close() + if err != nil { + return "", fmt.Errorf("read http body: %v", err) + } + return string(b), nil +} + +func addQueryToUri(rawurl string, kwargs map[string]string) string { + u, _ := url.ParseRequestURI(rawurl) + q := u.Query() + for k, v := range kwargs { + q.Add(k, v) + } + u.RawQuery = q.Encode() + return u.String() +} + +func encodeQueryToPayload(kwargs map[string]string) string { + payload := url.Values{} + for k, v := range kwargs { + payload.Set(k, v) + } + return payload.Encode() +} + +func readHTTPBodyToInt(resp *http.Response) (int64, error) { + b, err := readHTTPBody(resp.Body) + resp.Body.Close() + if err != nil { + return 0, fmt.Errorf("read http body: %v", err) + } + + n, err := strconv.ParseInt(string(b), 10, 64) + if err != nil { + return 0, fmt.Errorf("parse int: %v", err) + } + return n, nil +} + +func parseStrToInt(s string) int64 { + n, _ := strconv.ParseInt(s, 10, 64) + return n +} + +func md5File(f io.ReadSeeker) (string, error) { + offset, _ := f.Seek(0, 0) + defer f.Seek(offset, 0) + hash := md5.New() + if _, err := io.Copy(hash, f); err != nil { + return "", err + } + return fmt.Sprintf("%x", hash.Sum(nil)), nil +} + +func parseBodyToFileInfos(b []byte) (fInfos []*FileInfo) { + line := strings.Split(string(b), "\n") + for _, l := range line { + if len(l) == 0 { + continue + } + items := strings.Split(l, "\t") + if len(items) != 4 { + continue + } + + fInfos = append(fInfos, &FileInfo{ + Name: items[0], + IsDir: items[1] == "F", + Size: int64(parseStrToInt(items[2])), + Time: time.Unix(parseStrToInt(items[3]), 0), + }) + } + return +}