eson/requests
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge branch 'release/v1.2.0'

Browse Source
This commit is contained in:
huangsimin 2019-12-24 17:34:06 +08:00
commit a6e4e090e5
7 changed files with 153 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
base.go
View File

@ -4,9 +4,26 @@ import (
"bytes"
"errors"
"net/http"
"net/url"
"reflect"
)
// NotUpdateJar 不更新Cookie的jar
type notUpdateJar struct {
http.CookieJar
ProxyJar http.CookieJar
}
// SetCookies 设置cookie 这里将什么都不操作
func (jar *notUpdateJar) SetCookies(u *url.URL, cookies []*http.Cookie) {
}
// Cookies 返回Cookies
func (jar *notUpdateJar) Cookies(u *url.URL) []*http.Cookie {
return jar.ProxyJar.Cookies(u)
}
func buildBodyRequest(wf *Workflow) *http.Request {
var req *http.Request
var err error

7
go.mod
View File

@ -2,11 +2,12 @@ module github.com/474420502/requests
go 1.13
replace golang.org/x/net => github.com/golang/net v0.0.0-20191209160850-c0dbc17a3553
require (
github.com/474420502/gjson v1.1.3
github.com/elazarl/goproxy v0.0.0-2vi0190711103511-473e67f1d7d2
github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2 // indirect
github.com/elazarl/goproxy v0.0.0-20191011121108-aa519ddbe484
github.com/elazarl/goproxy/ext v0.0.0-20191011121108-aa519ddbe484 // indirect
github.com/tidwall/match v1.0.1 // indirect
golang.org/x/net v0.0.0-00010101000000-000000000000
)

13
go.sum
View File

@ -1,11 +1,16 @@
github.com/474420502/gjson v1.1.3 h1:rQxKNSFS8bM5iVVKKb9EHY1SS2k+EhzVNUXe2xSGn8o=
github.com/474420502/gjson v1.1.3/go.mod h1:mdAOevjPYIFWOE8CpejPHwoJCz96oNnuwhjhrAVeKaY=
github.com/elazarl/goproxy v0.0.0-20190711103511-473e67f1d7d2 h1:aZtFdDNWY/yH86JPR2WX/PN63635VsE/f/nXNPAbYxY=
github.com/elazarl/goproxy v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
github.com/elazarl/goproxy v0.0.0-20191011121108-aa519ddbe484 h1:pEtiCjIXx3RvGjlUJuCNxNOw0MNblyR9Wi+vJGBFh+8=
github.com/elazarl/goproxy v0.0.0-20191011121108-aa519ddbe484/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM=
github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2 h1:dWB6v3RcOy03t/bUadywsbyrQwCqZeNIEX6M1OtSZOM=
github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8=
github.com/golang/net v0.0.0-20190902185345-cdfb69ac37fc6fa907650654115ebebb3aae2087 h1:haK1T12C0CO79KUdu+ZzLL9+l9BwM9PRkd2/mQqdg8E=
github.com/golang/net v0.0.0-20190902185345-cdfb69ac37fc6fa907650654115ebebb3aae2087/go.mod h1:98y8FxUyMjTdJ5eOj/8vzuiVO14/dkJ98NYhEPG8QGY=
github.com/elazarl/goproxy/ext v0.0.0-20191011121108-aa519ddbe484 h1:aBgSK7SOjFeL1wfzfbIb/pkPpfRr6EP7b1Af4o/xkpU=
github.com/elazarl/goproxy/ext v0.0.0-20191011121108-aa519ddbe484/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8=
github.com/golang/net v0.0.0-20191209160850-c0dbc17a3553 h1:VChqxxcWJRNva3fY40u7VmDkRtWRBka24Djowq6iL30=
github.com/golang/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc=
github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc=
github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

29
session.go
View File

@ -102,7 +102,7 @@ type Session struct {
body IBody
client *http.Client
cookiejar http.CookieJar
cookiejar *cookiejar.Jar
transport *http.Transport
@ -176,12 +176,15 @@ const (
// CTLS 帐号认证
CTLS // user pwd
// CIsWithCookiejar 持久化 CookieJar true or false ; default = true
// CIsWithCookiejar 加载持久化 CookieJar true or false ; default = true
CIsWithCookiejar
// CIsDecompressNoAccept 解压 当response header 不存在 Accept-Encoding
// 很多特殊情景会不返回Accept-Encoding: Gzip. 如 不按照标准的网站
CIsDecompressNoAccept
// CIsKeepCookiejar 发生http请求时， 保持当前Cookiejar, 不被Set-Cookie更新; true of false; default false
CIsKeepCookiejar
)
// NewSession 创建Session
@ -198,10 +201,10 @@ func NewSession() *Session {
}
client.Jar = cjar
return &Session{client: client, body: NewBody(), transport: transport, auth: nil, cookiejar: client.Jar, Header: make(http.Header), Is: IsSetting{false}}
return &Session{client: client, body: NewBody(), transport: transport, auth: nil, cookiejar: cjar, Header: make(http.Header), Is: IsSetting{false}}
}
// SetConfig 设置配置
// SetConfig 设置配置, 如果使用高级方法需要了解这个可以配置的属性, 需要参考 TypeConfig
func (ses *Session) SetConfig(typeConfig TypeConfig, values interface{}) {
switch typeConfig {
@ -227,15 +230,6 @@ func (ses *Session) SetConfig(typeConfig TypeConfig, values interface{}) {
case CKeepAlives:
// println(ses.transport.DisableKeepAlives)
ses.transport.DisableKeepAlives = !values.(bool)
case CIsWithCookiejar:
v := values.(bool)
if v {
if ses.client.Jar == nil {
ses.client.Jar = ses.cookiejar
}
} else {
ses.client.Jar = nil
}
case CProxy:
switch v := values.(type) {
case string:
@ -271,6 +265,15 @@ func (ses *Session) SetConfig(typeConfig TypeConfig, values interface{}) {
case nil:
ses.auth = nil
}
case CIsKeepCookiejar:
v := values.(bool)
if v {
ses.client.Jar = &notUpdateJar{ProxyJar: ses.cookiejar}
} else {
ses.client.Jar = ses.cookiejar
}
default:
panic(errors.New("unknown typeConfig " + reflect.TypeOf(typeConfig).String()))
}

38
session_test.go
View File

@ -410,8 +410,6 @@ func TestSession_ConfigEx(t *testing.T) {
t.Error(err)
} else {
// jar.SetCookies(u, []*http.Cookie{&http.Cookie{Name: "Request", Value: "Cookiejar"}})
ses.SetConfig(CIsWithCookiejar, false)
ses.SetConfig(CIsWithCookiejar, true)
ses.SetCookies(u, []*http.Cookie{&http.Cookie{Name: "Request", Value: "Cookiejar"}, &http.Cookie{Name: "eson", Value: "bad"}})
resp, err = ses.Get("http://httpbin.org/get").Execute()
if err != nil {
@ -543,3 +541,39 @@ func TestSession_SetBasicAuth(t *testing.T) {
t.Error("code != 401, code = ", resp.GetStatue())
}
}
func TestSession_KeepCookiejar(t *testing.T) {
ses := NewSession()
ses.SetConfig(CIsKeepCookiejar, true)
wf := ses.Get("http://httpbin.org/cookies/set?c=1")
if _, err := wf.Execute(); err != nil {
t.Error("get http://httpbin.org/cookies/set?c=1 error")
}
if len(ses.GetCookies(wf.GetParsedURL())) > 0 {
t.Error(ses.GetCookies(wf.GetParsedURL()))
}
ses.SetConfig(CIsKeepCookiejar, false)
wf = ses.Get("http://httpbin.org/cookies/set?c=1")
if _, err := wf.Execute(); err != nil {
t.Error("get http://httpbin.org/cookies/set?c=1 error")
}
cookie := ses.GetCookies(wf.GetParsedURL())[0]
if cookie.Name != "c" || cookie.Value != "1" {
t.Error("cookie error, ", cookie)
}
ses.SetConfig(CIsKeepCookiejar, true)
wf = ses.Get("http://httpbin.org/cookies/set?c=2")
if _, err := wf.Execute(); err != nil {
t.Error("get http://httpbin.org/cookies/set?c=2 error")
}
cookie = ses.GetCookies(wf.GetParsedURL())[0]
// 值不会改变, 但是必须存在c=1旧值
if cookie.Name != "c" || cookie.Value != "1" {
t.Error("cookie error, ", cookie)
}
}

42
workflow.go
View File

@ -7,7 +7,10 @@ import (
"strings"
)
// Workflow 工作流 设计点: 这个并不影响Session的属性变化 如 NewWorkflow(ses, url).AddHeader() 对ses没影响
// Workflow 工作流
// 设计点: 1. 这个并不影响Session的属性变化 如 NewWorkflow(ses, url).AddHeader() 对ses没影响
// 没影响指: 并不会在Session上永久加上Header.
// 2.
type Workflow struct {
session *Session
ParsedURL *url.URL
@ -20,7 +23,7 @@ type Workflow struct {
// NewWorkflow new and init workflow
func NewWorkflow(ses *Session, urlstr string) *Workflow {
wf := &Workflow{}
wf.SwitchSession(ses)
wf.SetSession(ses)
wf.SetRawURL(urlstr)
wf.Body = NewBody()
@ -29,8 +32,8 @@ func NewWorkflow(ses *Session, urlstr string) *Workflow {
return wf
}
// SwitchSession 替换Session
func (wf *Workflow) SwitchSession(ses *Session) {
// SetSession 替换Session
func (wf *Workflow) SetSession(ses *Session) {
wf.session = ses
}
@ -69,6 +72,23 @@ func (wf *Workflow) DelHeader(key string) *Workflow {
return wf
}
// GetCookies 获取 Key(http.Cookie.Name) Value Cookie 如果没有返回nil
func (wf *Workflow) GetCookies() []*http.Cookie {
var result []*http.Cookie
for _, v := range wf.Cookies {
result = append(result, v)
}
return result
}
// GetCookie 获取 Key(http.Cookie.Name) Value Cookie 如果没有返回nil
func (wf *Workflow) GetCookie(key string) *http.Cookie {
if cookie, ok := wf.Cookies[key]; ok {
return cookie
}
return nil
}
// AddCookie 添加Cookie
func (wf *Workflow) AddCookie(c *http.Cookie) *Workflow {
wf.Cookies[c.Name] = c
@ -140,7 +160,7 @@ func (wf *Workflow) GetQuery() url.Values {
return wf.ParsedURL.Query()
}
// GetCombineQuery 获取Query参数
// GetCombineQuery 获取session 和 workflow的 Query参数集
func (wf *Workflow) GetCombineQuery() url.Values {
if wf.ParsedURL != nil {
vs := wf.ParsedURL.Query()
@ -316,6 +336,18 @@ func setTempCookieRequest(req *http.Request, wf *Workflow) {
}
}
// UpdateCookiesToSession 更新cookies 到 session cookies上 意义不大
// 参数isCoverMode: true 覆盖模式, 在原Session值上覆盖. 如果没对应的Key,不进行覆盖.
// 参数isCoverMode: false 非覆盖模式, 直接从新生成与workflow上的Session对应的Cookies
// func (wf *Workflow) UpdateCookiesToSession(isCoverMode bool) {
// if wf.session != nil {
// if !isCoverMode {
// wf.session.ClearCookies()
// }
// wf.session.SetCookies(wf.GetParsedURL(), wf.GetCookies())
// }
// }
// Execute 执行
func (wf *Workflow) Execute() (*Response, error) {

34
workflow_test.go
View File

@ -304,3 +304,37 @@ func TestWorkflow_Body(t *testing.T) {
// wf.SetBody(body)
// resp, _ = wf.Execute()
}
func TestWorkflow_UpdateCookiesToSession(t *testing.T) {
ses := NewSession()
ses.SetConfig(CIsKeepCookiejar, true)
wf := ses.Get("http://httpbin.org/cookies/set?c=1")
wf.AddKVCookie("a", "1").AddKVCookie("b", "2")
_, err := wf.Execute()
if err != nil {
t.Error("cookies set error", err)
}
if c := wf.GetCookie("a"); c.Name != "a" {
t.Error("cookie error")
} else {
if c.Value != "1" {
t.Error("cookie error")
}
}
if c := wf.GetCookie("b"); c.Name != "b" {
t.Error("cookie error")
} else {
if c.Value != "2" {
t.Error("cookie error")
}
}
if len(ses.GetCookies(wf.GetParsedURL())) > 0 {
t.Error("CIsKeepCookiejar error")
}
}