20 Commits

Author SHA1 Message Date
965d1e6d16 fix: FromResponse not return error. 2018-12-23 00:15:24 +08:00
a0d8830242 golang open files 是keepalives的问题. 2018-12-18 03:16:17 +08:00
823e5e0bcf add default keepalvies false 2018-12-18 02:17:04 +08:00
30876ae2bf test fd 2018-12-18 01:46:41 +08:00
75fc71ccb4 丑陋的解压 2018-12-18 01:42:38 +08:00
e165b6a0a0 test open files 2018-12-18 01:36:18 +08:00
2307e6eb10 req is nil with Client.Do 2018-12-18 01:09:29 +08:00
28d4c9a297 fix code 2018-12-18 01:06:21 +08:00
18de9c2c1f req body close 2018-12-18 01:04:32 +08:00
1f673f991f delete body copy 2018-12-18 00:57:31 +08:00
2e315b1b52 add request close 2018-12-18 00:50:09 +08:00
16859fc78e add close Reader 2018-12-18 00:21:27 +08:00
huangsimin
72491204d6 Merge branch 'master' of http://474420502.top/eson/requests 2018-11-23 15:47:18 +08:00
huangsimin
84ae0a710e accept iobody the type is string; 2018-11-23 15:45:34 +08:00
12cf8d58ba change InsecureSkipVerify logic! 2018-11-23 00:50:54 +08:00
huangsimin
a6217886dc fix SetHeader Replace src header(copy all header in a new http.Header) 2018-11-21 17:43:22 +08:00
huangsimin
0c71408d1f add file close; add multipart close; 2018-11-21 17:03:31 +08:00
huangsimin
9d6dee25c9 IOBODY -> GETIOBODY 2018-11-21 16:29:13 +08:00
huangsimin
8a6e29bc8a change body method; -> AutoSetBody; add method SetBody; 2018-11-21 15:54:06 +08:00
huangsimin
c4a12f9314 optimization code 2018-11-21 15:43:52 +08:00
7 changed files with 98 additions and 65 deletions

12
base.go
View File

@@ -12,17 +12,19 @@ func buildBodyRequest(wf *Workflow) *http.Request {
var err error var err error
contentType := "" contentType := ""
if wf.Body.IOBody() == nil { if wf.Body.GetIOBody() == nil {
req, err = http.NewRequest(wf.Method, wf.GetRawURL(), nil) req, err = http.NewRequest(wf.Method, wf.GetRawURL(), nil)
} else { } else {
var bodybuf *bytes.Buffer var bodybuf *bytes.Buffer
switch wf.Body.IOBody().(type) { switch wf.Body.GetIOBody().(type) {
case []byte: case []byte:
bodybuf = bytes.NewBuffer(wf.Body.IOBody().([]byte)) bodybuf = bytes.NewBuffer(wf.Body.GetIOBody().([]byte))
case string:
bodybuf = bytes.NewBuffer([]byte(wf.Body.GetIOBody().(string)))
case *bytes.Buffer: case *bytes.Buffer:
bodybuf = bytes.NewBuffer(wf.Body.IOBody().(*bytes.Buffer).Bytes()) bodybuf = bytes.NewBuffer(wf.Body.GetIOBody().(*bytes.Buffer).Bytes())
default: default:
panic(errors.New("the type is not exist, type is" + reflect.TypeOf(wf.Body.IOBody).String())) panic(errors.New("the type is not exist, type is " + reflect.TypeOf(wf.Body.GetIOBody()).String()))
} }
req, err = http.NewRequest(wf.Method, wf.GetRawURL(), bodybuf) req, err = http.NewRequest(wf.Method, wf.GetRawURL(), bodybuf)
} }

View File

@@ -15,6 +15,11 @@ func writeFormUploadFile(mwriter *multipart.Writer, ufile *UploadFile) {
log.Panic(err) log.Panic(err)
} }
io.Copy(part, ufile.FileReaderCloser) io.Copy(part, ufile.FileReaderCloser)
err = ufile.FileReaderCloser.Close()
if err != nil {
panic(err)
}
} }
func createMultipart(postParams IBody, params []interface{}) { func createMultipart(postParams IBody, params []interface{}) {
@@ -23,8 +28,6 @@ func createMultipart(postParams IBody, params []interface{}) {
body := &bytes.Buffer{} body := &bytes.Buffer{}
mwriter := multipart.NewWriter(body) mwriter := multipart.NewWriter(body)
defer mwriter.Close()
for _, iparam := range params[0 : plen-1] { for _, iparam := range params[0 : plen-1] {
switch param := iparam.(type) { switch param := iparam.(type) {
case *UploadFile: case *UploadFile:
@@ -87,4 +90,9 @@ func createMultipart(postParams IBody, params []interface{}) {
postParams.AddContentType(mwriter.FormDataContentType()) postParams.AddContentType(mwriter.FormDataContentType())
postParams.SetIOBody(body) postParams.SetIOBody(body)
err := mwriter.Close()
if err != nil {
panic(err)
}
} }

View File

@@ -4,7 +4,6 @@ import (
"bytes" "bytes"
"compress/gzip" "compress/gzip"
"compress/zlib" "compress/zlib"
"io"
"io/ioutil" "io/ioutil"
"net/http" "net/http"
) )
@@ -21,34 +20,29 @@ func FromHTTPResponse(resp *http.Response) (*Response, error) {
// 复制response 返回内容 并且测试是否有解压的需求 // 复制response 返回内容 并且测试是否有解压的需求
srcbuf, err := ioutil.ReadAll(resp.Body) srcbuf, err := ioutil.ReadAll(resp.Body)
if err != nil { if err != nil {
panic(err) return nil, err
} }
resp.Body.Close() resp.Body.Close()
cbuf := bytes.NewBuffer([]byte{}) content := ""
_, err = io.Copy(cbuf, bytes.NewReader(srcbuf))
if err != nil {
panic(err)
}
resp.Body = ioutil.NopCloser(cbuf)
content := string(srcbuf)
srcReader := bytes.NewReader(srcbuf) srcReader := bytes.NewReader(srcbuf)
if r, err := gzip.NewReader(srcReader); err == nil { if r, err := gzip.NewReader(srcReader); err == nil {
defer r.Close()
buf, err := ioutil.ReadAll(r) buf, err := ioutil.ReadAll(r)
if err != nil { if err != nil {
panic(err) return nil, err
} }
content = string(buf) content = string(buf)
} else if r, err := zlib.NewReader(srcReader); err == nil { } else if r, err := zlib.NewReader(srcReader); err == nil {
defer r.Close()
buf, err := ioutil.ReadAll(r) buf, err := ioutil.ReadAll(r)
if err != nil { if err != nil {
panic(err) return nil, err
} }
content = string(buf) content = string(buf)
} else {
content = string(srcbuf)
} }
return &Response{DContent: content, GResponse: resp}, nil return &Response{DContent: content, GResponse: resp}, nil

View File

@@ -36,8 +36,8 @@ func (body *Body) SetIOBody(iobody interface{}) {
body.ioBody = iobody body.ioBody = iobody
} }
// IOBody 获取ioBody值 // GetIOBody 获取ioBody值
func (body *Body) IOBody() interface{} { func (body *Body) GetIOBody() interface{} {
return body.ioBody return body.ioBody
} }
@@ -70,16 +70,15 @@ func (body *Body) AddContentType(ct string) {
// IBody 相关参数结构 // IBody 相关参数结构
type IBody interface { type IBody interface {
// GetIOBody 获取iobody data
// Query map[string][]string GetIOBody() interface{}
IOBody() interface{} // SetIOBody 设置iobody data
// SetIOBody
SetIOBody(iobody interface{}) SetIOBody(iobody interface{})
// Files []UploadFile // ContentType 返回包括 Prefix 所有的ContentType
ContentType() string ContentType() string
// AppendContent // AppendContent
AddContentType(ct string) AddContentType(ct string)
// AddPrefix 添加 Prefix // SetPrefix 设置 Prefix; 唯一前缀
SetPrefix(ct string) SetPrefix(ct string)
} }
@@ -155,6 +154,9 @@ const (
// CDialTimeout 一个Connect过程的Timeout // CDialTimeout 一个Connect过程的Timeout
CDialTimeout // 支持time.Duration 和 int(秒为单位) CDialTimeout // 支持time.Duration 和 int(秒为单位)
// CKeepAlives 默认不KeepAlives, 容易被一直KeepAlives 没关闭链接
CKeepAlives
// CProxy 代理链接 // CProxy 代理链接
CProxy // http, https, socks5 CProxy // http, https, socks5
@@ -174,7 +176,9 @@ const (
// NewSession 创建Session // NewSession 创建Session
func NewSession() *Session { func NewSession() *Session {
client := &http.Client{} client := &http.Client{}
transport := &http.Transport{DisableCompression: true} transport := &http.Transport{DisableCompression: true, DisableKeepAlives: true}
EnsureTransporterFinalized(transport)
client.Transport = transport client.Transport = transport
cjar, err := cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List}) cjar, err := cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List})
@@ -207,6 +211,8 @@ func (ses *Session) SetConfig(typeConfig TypeConfig, values interface{}) {
} }
case CDialTimeout: case CDialTimeout:
// 没时间实现这些小细节 // 没时间实现这些小细节
case CKeepAlives:
ses.transport.DisableKeepAlives = !values.(bool)
case CCookiejar: case CCookiejar:
v := values.(bool) v := values.(bool)
if v { if v {
@@ -230,7 +236,7 @@ func (ses *Session) SetConfig(typeConfig TypeConfig, values interface{}) {
ses.transport.Proxy = nil ses.transport.Proxy = nil
} }
case CInsecure: case CInsecure:
ses.transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: !values.(bool)} ses.transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: values.(bool)}
case CTLS: case CTLS:
ses.transport.TLSClientConfig = values.(*tls.Config) ses.transport.TLSClientConfig = values.(*tls.Config)
case CBasicAuth: case CBasicAuth:
@@ -308,6 +314,13 @@ func (ses *Session) ClearCookies() {
ses.client.Jar = ses.cookiejar ses.client.Jar = ses.cookiejar
} }
// Head 请求
func (ses *Session) Head(url string) *Workflow {
wf := NewWorkflow(ses, url)
wf.Method = "HEAD"
return wf
}
// Get 请求 // Get 请求
func (ses *Session) Get(url string) *Workflow { func (ses *Session) Get(url string) *Workflow {
wf := NewWorkflow(ses, url) wf := NewWorkflow(ses, url)
@@ -336,20 +349,6 @@ func (ses *Session) Patch(url string) *Workflow {
return wf return wf
} }
// Delete 请求
func (ses *Session) Delete(url string) *Workflow {
wf := NewWorkflow(ses, url)
wf.Method = "DELETE"
return wf
}
// Head 请求
func (ses *Session) Head(url string) *Workflow {
wf := NewWorkflow(ses, url)
wf.Method = "HEAD"
return wf
}
// Options 请求 // Options 请求
func (ses *Session) Options(url string) *Workflow { func (ses *Session) Options(url string) *Workflow {
wf := NewWorkflow(ses, url) wf := NewWorkflow(ses, url)
@@ -357,16 +356,22 @@ func (ses *Session) Options(url string) *Workflow {
return wf return wf
} }
// CloseIdleConnections closes the idle connections that a session client may make use of // Delete 请求
// 从levigross/grequests 借鉴 func (ses *Session) Delete(url string) *Workflow {
func (ses *Session) CloseIdleConnections() { wf := NewWorkflow(ses, url)
ses.client.Transport.(*http.Transport).CloseIdleConnections() wf.Method = "DELETE"
return wf
} }
// // CloseIdleConnections closes the idle connections that a session client may make use of
// // 从levigross/grequests 借鉴
// func (ses *Session) CloseIdleConnections() {
// ses.client.Transport.(*http.Transport).CloseIdleConnections()
// }
// EnsureTransporterFinalized will ensure that when the HTTP client is GCed // EnsureTransporterFinalized will ensure that when the HTTP client is GCed
// the runtime will close the idle connections (so that they won't leak) // the runtime will close the idle connections (so that they won't leak)
// this function was adopted from Hashicorp's go-cleanhttp package // this function was adopted from Hashicorp's go-cleanhttp package
// 暂时不用, 标记到以后是否起作用
func EnsureTransporterFinalized(httpTransport *http.Transport) { func EnsureTransporterFinalized(httpTransport *http.Transport) {
runtime.SetFinalizer(&httpTransport, func(transportInt **http.Transport) { runtime.SetFinalizer(&httpTransport, func(transportInt **http.Transport) {
(*transportInt).CloseIdleConnections() (*transportInt).CloseIdleConnections()

View File

@@ -76,7 +76,7 @@ func TestSession_Post(t *testing.T) {
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
ses := NewSession() ses := NewSession()
got, err := ses.Post("http://httpbin.org/post").SetBody(tt.args.params...).Execute() got, err := ses.Post("http://httpbin.org/post").AutoSetBody(tt.args.params...).Execute()
if err != nil { if err != nil {
t.Errorf("Metchod error = %v", err) t.Errorf("Metchod error = %v", err)
@@ -127,7 +127,7 @@ func TestSession_Setparams(t *testing.T) {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
ses := NewSession() ses := NewSession()
got, err := ses.Post("http://httpbin.org/post").SetBody(tt.args.params...).Execute() got, err := ses.Post("http://httpbin.org/post").AutoSetBody(tt.args.params...).Execute()
if (err != nil) != tt.wantErr { if (err != nil) != tt.wantErr {
t.Errorf("Metchod error = %v, wantErr %v", err, tt.wantErr) t.Errorf("Metchod error = %v, wantErr %v", err, tt.wantErr)
return return
@@ -169,7 +169,7 @@ func TestSession_PostUploadFile(t *testing.T) {
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
ses := NewSession() ses := NewSession()
got, err := ses.Post("http://httpbin.org/post").SetBody(tt.args.params, TypeFormData).Execute() got, err := ses.Post("http://httpbin.org/post").AutoSetBody(tt.args.params, TypeFormData).Execute()
if err != nil { if err != nil {
t.Errorf("Metchod error = %v", err) t.Errorf("Metchod error = %v", err)
@@ -213,7 +213,7 @@ func TestSession_Put(t *testing.T) {
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
ses := NewSession() ses := NewSession()
got, err := ses.Put("http://httpbin.org/put").SetBody(tt.args.params, TypeFormData).Execute() got, err := ses.Put("http://httpbin.org/put").AutoSetBody(tt.args.params, TypeFormData).Execute()
if err != nil { if err != nil {
t.Errorf("Metchod error = %v", err) t.Errorf("Metchod error = %v", err)
@@ -257,7 +257,7 @@ func TestSession_Patch(t *testing.T) {
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
ses := NewSession() ses := NewSession()
got, err := ses.Patch("http://httpbin.org/patch").SetBody(tt.args.params, TypeFormData).Execute() got, err := ses.Patch("http://httpbin.org/patch").AutoSetBody(tt.args.params, TypeFormData).Execute()
if err != nil { if err != nil {
t.Errorf("Metchod error = %v", err) t.Errorf("Metchod error = %v", err)
@@ -319,7 +319,7 @@ func TestSession_SetConfig(t *testing.T) {
func TestSession_SetConfigInsecure(t *testing.T) { func TestSession_SetConfigInsecure(t *testing.T) {
ses := NewSession() ses := NewSession()
ses.SetConfig(CInsecure, false) ses.SetConfig(CInsecure, true)
for _, badSSL := range []string{ for _, badSSL := range []string{
"https://self-signed.badssl.com/", "https://self-signed.badssl.com/",

View File

@@ -17,13 +17,11 @@ type UploadFile struct {
// UploadFileFromPath 从本地文件获取上传文件 // UploadFileFromPath 从本地文件获取上传文件
func UploadFileFromPath(fileName string) (*UploadFile, error) { func UploadFileFromPath(fileName string) (*UploadFile, error) {
fd, err := os.Open(fileName) fd, err := os.Open(fileName)
if err != nil { if err != nil {
return nil, err return nil, err
} }
return &UploadFile{FileReaderCloser: fd, FileName: fileName}, nil return &UploadFile{FileReaderCloser: fd, FileName: fileName}, nil
} }
// UploadFileFromGlob 根据Glob从本地文件获取上传文件 // UploadFileFromGlob 根据Glob从本地文件获取上传文件

View File

@@ -41,9 +41,16 @@ func (wf *Workflow) AddHeader(key, value string) *Workflow {
return wf return wf
} }
// SetHeader 设置头信息 这样构造 []string{value} Get方法从Header参数上获取 // SetHeader 设置完全替换原有Header
func (wf *Workflow) SetHeader(key, value string) *Workflow { func (wf *Workflow) SetHeader(header http.Header) *Workflow {
wf.Header.Set(key, value) wf.Header = make(http.Header)
for k, HValues := range header {
var newHValues []string
for _, value := range HValues {
newHValues = append(newHValues, value)
}
wf.Header[k] = newHValues
}
return wf return wf
} }
@@ -69,6 +76,14 @@ func (wf *Workflow) AddCookie(c *http.Cookie) *Workflow {
return wf return wf
} }
// AddCookies 添加[]*http.Cookie
func (wf *Workflow) AddCookies(cookies []*http.Cookie) *Workflow {
for _, c := range cookies {
wf.AddCookie(c)
}
return wf
}
// AddKVCookie 添加 以 key value 的 Cookie // AddKVCookie 添加 以 key value 的 Cookie
func (wf *Workflow) AddKVCookie(name, value string) *Workflow { func (wf *Workflow) AddKVCookie(name, value string) *Workflow {
wf.Cookies[name] = &http.Cookie{Name: name, Value: value} wf.Cookies[name] = &http.Cookie{Name: name, Value: value}
@@ -185,7 +200,18 @@ func (wf *Workflow) SetURLRawPath(path string) *Workflow {
} }
// SetBody 参数设置 // SetBody 参数设置
func (wf *Workflow) SetBody(params ...interface{}) *Workflow { func (wf *Workflow) SetBody(body IBody) *Workflow {
wf.Body = body
return wf
}
// GetBody 参数设置
func (wf *Workflow) GetBody(body IBody) IBody {
return wf.Body
}
// AutoSetBody 参数设置
func (wf *Workflow) AutoSetBody(params ...interface{}) *Workflow {
if params != nil { if params != nil {
plen := len(params) plen := len(params)
@@ -194,11 +220,10 @@ func (wf *Workflow) SetBody(params ...interface{}) *Workflow {
if plen >= 2 { if plen >= 2 {
t := params[plen-1] t := params[plen-1]
defaultContentType = t.(string) defaultContentType = t.(string)
wf.Body.SetPrefix(defaultContentType)
} else {
wf.Body.SetPrefix(defaultContentType)
} }
wf.Body.SetPrefix(defaultContentType)
if defaultContentType == TypeFormData { if defaultContentType == TypeFormData {
createMultipart(wf.Body, params) createMultipart(wf.Body, params)
} else { } else {
@@ -286,5 +311,6 @@ func (wf *Workflow) Execute() (*Response, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
return FromHTTPResponse(resp) return FromHTTPResponse(resp)
} }