test/chromeworker_client
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Compare commits

base: test:master
Branches Tags
test:master
test:v0.2.3
test:v0.2.2
test:v0.2.1
test:v0.2.0
test:v0.1.1
test:v0.1.0
test:v0.0.4
test:v0.0.3
test:v0.0.2
test:v0.0.1
..
compare: test:v0.0.4
Branches Tags
test:master
test:v0.2.3
test:v0.2.2
test:v0.2.1
test:v0.2.0
test:v0.1.1
test:v0.1.0
test:v0.0.4
test:v0.0.3
test:v0.0.2
test:v0.0.1

No commits in common. "master" and "v0.0.4" have entirely different histories.
master ... v0.0.4

5 changed files with 83 additions and 187 deletions
Show Stats Expand all files Collapse all files

1
.gitignore vendored
View File

@ -1 +0,0 @@
debug.test

45
carray.go
View File

@ -1,45 +0,0 @@
package cwclient
import "time"
// Carray 每次url请求都会携带一个值. 可以为nil
type Carray struct {
data interface{} // 携带的数据. 每次请求唯一.
hash string // 每次请求唯一hash. 标识.
expire time.Time // 过期时间
}
// GetExpire Get return expire time.Time
func (undefined *Carray) GetExpire() time.Time {
return undefined.expire
}
// SetExpire Set expire time.Time
func (undefined *Carray) SetExpire(expire time.Time) {
undefined.expire = expire
}
// GetHash Get return hash string
func (undefined *Carray) GetHash() string {
return undefined.hash
}
// SetHash Set hash string
func (undefined *Carray) SetHash(hash string) {
undefined.hash = hash
}
// GetData Get return data interface{}
func (undefined *Carray) GetData() interface{} {
return undefined.data
}
// SetData Set data interface{}
func (undefined *Carray) SetData(data interface{}) {
undefined.data = data
}
// Timeup 判断是否过期,超时
func (undefined *Carray) Timeup() bool {
return time.Now().Before(undefined.expire)
}

127
client.go
View File

@ -8,39 +8,23 @@ import (
"net" "net"
"net/http" "net/http"
"net/url" "net/url"
"os"
"strconv"
"sync" "sync"
"time"
"github.com/gin-gonic/gin" "github.com/gin-gonic/gin"
"github.com/google/uuid" "github.com/google/uuid"
) )
func init() {
log.SetFlags(log.Llongfile | log.LstdFlags)
}
// CallbackContext Callback上下文
type CallbackContext struct {
TaskID string
Content string
Error error
Carry interface{} // 传递的参数.
}
// Callback 发送代理连接获取内容后的回调函数 // Callback 发送代理连接获取内容后的回调函数
type Callback struct { type Callback struct {
label string label string
hash string hash string
Do func(cxt *CallbackContext) Do func(tid, content string)
} }
// Client 客户端 // Client 客户端
type Client struct { type Client struct {
chromeProxyAddr string chromeProxyAddr string
carrayCache sync.Map
register sync.Map register sync.Map
host string host string
@ -54,37 +38,9 @@ type Client struct {
// Label 区分不同任务类型 // Label 区分不同任务类型
type Label struct { type Label struct {
label string label string
conditionJS string
retry string
waitcapture string
configlock sync.Mutex
cli *Client cli *Client
} }
// GetWaitime Get return waitime int
func (l *Label) GetWaitime() int {
r, _ := strconv.Atoi(l.waitcapture)
return r
}
// SetWaitime Set waitime int
func (l *Label) SetWaitime(waitime int) {
l.waitcapture = strconv.Itoa(waitime)
}
// GetRetry Get return retry int
func (l *Label) GetRetry() int {
r, _ := strconv.Atoi(l.retry)
return r
}
// SetRetry Set retry int
func (l *Label) SetRetry(retry int) {
l.retry = strconv.Itoa(retry)
}
// GetHash 根据label获取hash路径 // GetHash 根据label获取hash路径
func (l *Label) GetHash(label string) string { func (l *Label) GetHash(label string) string {
if cb, ok := l.cli.register.Load(label); ok { if cb, ok := l.cli.register.Load(label); ok {
@ -101,32 +57,9 @@ func (l *Label) GetLabel(hash string) string {
return "" return ""
} }
// Open 缓存了Label值. 每次调用少了label传参. carray每次都会给一次请求回调传入参数. // Open 缓存了Label值. 每次调用少了label传参
func (l *Label) Open(urlstr string, carray interface{}) (bodyRes string, ok bool) { func (l *Label) Open(urlstr string) (bodyRes string, ok bool) {
return l.cli.open(l, urlstr, carray) return l.cli.Open(l.label, urlstr)
}
// SetContentCondition 设置识别到的内容条件. js代码. 必须是一个函数. 命名可以随意. 返回bool
func (l *Label) SetContentCondition(jsScript string) {
l.configlock.Lock()
defer l.configlock.Unlock()
l.conditionJS = jsScript
}
// SetContentConditionFromFile 设置识别到的内容条件. js代码. 从js文件 必须是一个函数. 命名可以随意. 返回bool
func (l *Label) SetContentConditionFromFile(jsScriptFile string) {
f, err := os.Open(jsScriptFile)
if err != nil {
panic(err)
}
data, err := ioutil.ReadAll(f)
if err != nil {
panic(err)
}
l.configlock.Lock()
defer l.configlock.Unlock()
// log.Println(string(data))
l.conditionJS = string(data)
} }
// GetPort Get return port string. default random. // GetPort Get return port string. default random.
@ -150,14 +83,14 @@ func (cli *Client) SetHost(host string) {
} }
// Register 注册基础信息 // Register 注册基础信息
func (cli *Client) Register(label string, callback func(cxt *CallbackContext)) *Label { func (cli *Client) Register(label string, callback func(tid, content string)) *Label {
cb := Callback{Do: callback, hash: uuid.New().String()} cb := Callback{Do: callback, hash: uuid.New().String()}
if _, ok := cli.register.Load(label); ok { if _, ok := cli.register.Load(label); ok {
log.Panic("label: ", label, " is exists") log.Panic("label: ", label, " is exists")
} }
cli.register.Store(label, cb) cli.register.Store(label, cb)
cli.register.Store(cb.hash, cb) cli.register.Store(cb.hash, cb)
l := &Label{label: label, cli: cli, waitcapture: "6000", retry: "1"} l := &Label{label: label, cli: cli}
return l return l
} }
@ -190,26 +123,7 @@ func (cli *Client) Connect() {
callback := f.(Callback) callback := f.(Callback)
if tid, ok := c.GetPostForm("taskid"); ok { if tid, ok := c.GetPostForm("taskid"); ok {
content := c.PostForm("content") content := c.PostForm("content")
errorStr := c.PostForm("error") callback.Do(tid, content)
carrayhash := c.PostForm("carrayhash")
var carray interface{}
if icarray, ok := cli.carrayCache.Load(carrayhash); ok {
carray = icarray.(*Carray).data
}
var err error = nil
if errorStr != "" {
err = fmt.Errorf(errorStr)
}
cxt := &CallbackContext{
TaskID: tid,
Content: content,
Error: err,
Carry: carray,
}
callback.Do(cxt)
} }
} }
}) })
@ -241,36 +155,19 @@ func (cli *Client) Disconnect() {
} }
} }
// open 请求完url后 调用不同label注册的回调函数. bodyRes 请求后服务器返回的基础信息. 如果不需要debug一般不需要使用. // Open 请求完url后 调用不同label注册的回调函数
func (cli *Client) open(label *Label, urlstr string, carray interface{}) (bodyRes string, ok bool) { func (cli *Client) Open(label, urlstr string) (bodyRes string, ok bool) {
// urlstr = "https://playerduo.com/api/playerDuo-service-v2/rip113?lang=en&deviceType=browser" // urlstr = "https://playerduo.com/api/playerDuo-service-v2/rip113?lang=en&deviceType=browser"
if cli.server == nil { if cli.server == nil {
panic("client is not connect. Client.Connect() ? ") panic("client is not connect. Client.Connect() ? ")
} }
if callback, ok := cli.register.Load(label.label); ok { if callback, ok := cli.register.Load(label); ok {
data := url.Values{} data := url.Values{}
data["url"] = []string{urlstr} data["url"] = []string{urlstr}
data["callback"] = []string{cli.host + ":" + cli.port + "/" + callback.(Callback).hash} data["callback"] = []string{cli.host + ":" + cli.port + "/" + callback.(Callback).hash}
data["label"] = []string{label.label} data["label"] = []string{label}
if carray != nil {
carrayhash := uuid.New().String()
c := &Carray{hash: carrayhash, data: carray, expire: time.Now().Add(time.Minute * 2)}
label.cli.carrayCache.Store(carrayhash, c)
data["carrayhash"] = []string{carrayhash}
}
func() {
label.configlock.Lock()
defer label.configlock.Unlock()
data["content_condition"] = []string{label.conditionJS}
data["waitcapture"] = []string{label.waitcapture}
data["retry"] = []string{label.retry}
}()
resp, err := http.DefaultClient.PostForm(cli.chromeProxyAddr+"/task/put", data) resp, err := http.DefaultClient.PostForm(cli.chromeProxyAddr+"/task/put", data)
if err != nil { if err != nil {
panic(err) panic(err)
@ -283,7 +180,7 @@ func (cli *Client) open(label *Label, urlstr string, carray interface{}) (bodyRe
return string(bodyRes), true return string(bodyRes), true
} }
log.Printf("label: %s is not exists", label.label) log.Printf("label: %s is not exists", label)
return "", false return "", false
} }

87
client_test.go
View File

@ -1,36 +1,87 @@
package cwclient package cwclient
import ( import (
"encoding/json" "fmt"
"log" "log"
"net/http" "net/http"
"os"
"testing" "testing"
"time"
"github.com/tebeka/selenium"
"github.com/tebeka/selenium/chrome"
) )
func TestPort(t *testing.T) { func TestPort(t *testing.T) {
cli := New("http://localhost:7123") cli := New("http://localhost:7123")
ltest := cli.Register("test", func(tid, content string) {
log.Println(tid, content)
ltest := cli.Register("test", func(cxt *CallbackContext) {
if cxt.Error != nil {
log.Println("error:", cxt.Error)
} else {
log.Println(cxt.TaskID, cxt.Content, cxt.Carry)
}
}) })
ltest.SetContentConditionFromFile("example.js")
cli.Connect() cli.Connect()
log.Println(ltest.Open("https://playerduo.com/api/playerDuo-service-v2/rip113?lang=en&deviceType=browser", nil)) log.Println(ltest.Open("https://playerduo.com/api/playerDuo-service-v2/rip113?lang=en&deviceType=browser"))
// log.Println(ltest.Open("https://playerduo.com/api/playerDuo-service-v2/yanngu?lang=en&deviceType=browser", "213")) log.Println(ltest.Open("https://playerduo.com/api/playerDuo-service-v2/yanngu?lang=en&deviceType=browser"))
// log.Println(ltest.Open("https://playerduo.com/api/playerDuo-service-v2/rip113?lang=en&deviceType=browser")) log.Println(ltest.Open("https://playerduo.com/api/playerDuo-service-v2/rip113?lang=en&deviceType=browser"))
// log.Println(ltest.Open("https://playerduo.com/api/playerDuo-service-v2/yanngu?lang=en&deviceType=browser")) log.Println(ltest.Open("https://playerduo.com/api/playerDuo-service-v2/yanngu?lang=en&deviceType=browser"))
http.ListenAndServe(":4233", nil) http.ListenAndServe(":42311", nil)
} }
func TestChrome(t *testing.T) { func TestChrome(t *testing.T) {
port := 41331
var a = make(map[string]interface{}) var err error
a["as"] = 123 caps := selenium.Capabilities{"browserName": "chrome"}
data, _ := json.Marshal(a["as"])
log.Println(string(data), a["as"]) chromecaps := chrome.Capabilities{}
// for _, epath := range []string{"../../../crx/myblock.crx", "../../crx/myblock.crx", "./crx/myblock.crx"} {
// _, err := os.Stat(epath)
// if err == nil {
// err := chromecaps.AddExtension(epath)
// if err != nil {
// panic(err)
// }
// break
// }
// }
chromecaps.Args = append(chromecaps.Args, "--disable-blink-features=AutomationControlled")
chromecaps.Args = append(chromecaps.Args, "user-agent=Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.102 Safari/537.36")
if proxy := os.Getenv("chrome_proxy"); proxy != "" {
log.Println("proxy-server", proxy)
chromecaps.Args = append(chromecaps.Args, "--proxy-server="+proxy)
}
if proxy := os.Getenv("pac_proxy"); proxy != "" {
log.Println("--proxy-pac-url=" + proxy)
chromecaps.Args = append(chromecaps.Args, "--proxy-pac-url="+proxy)
}
// chromecaps.Args = append(chromecaps.Args, "--proxy-pac-url=http://127.0.0.1:1081/pac")
chromecaps.Args = append(chromecaps.Args, "--disk-cache-dir=/tmp/chromedriver-cache")
chromecaps.Args = append(chromecaps.Args, "--user-data-dir=/tmp/chromedriver-userdata")
chromecaps.Args = append(chromecaps.Args, "--auto-open-devtools-for-tabs")
chromecaps.Args = append(chromecaps.Args, "--disable-gpu", "--disable-images", "--start-maximized", "--disable-infobars")
// chromecaps.Args = append(chromecaps.Args, "--headless")
chromecaps.Args = append(chromecaps.Args, "--no-sandbox")
chromecaps.Args = append(chromecaps.Args, "--disable-dev-shm-usage", "--mute-audio", "--safebrowsing-disable-auto-update")
chromecaps.ExcludeSwitches = append(chromecaps.ExcludeSwitches, "enable-automation")
caps.AddChrome(chromecaps)
_, err = selenium.NewChromeDriverService("/usr/bin/chromedriver", port)
if err != nil {
panic(err)
}
wd, err := selenium.NewRemote(caps, fmt.Sprintf("http://localhost:%d/wd/hub", 9222))
if err != nil {
panic(err)
}
err = wd.Get("https://www.amazon.com/")
if err != nil {
log.Println(err)
}
time.Sleep(time.Second * 10)
} }

6
example.js
View File

@ -1,6 +0,0 @@
condition = function () {
var href = window.location.href;
var content = document.documentElement.innerHTML;
return href.startsWith("https://playerduo.com") && content.startsWith('<head></head><body><pre style="word-wrap: break-word; white-space: pre-wrap;">')
}