feat(config): add retry and waitcapture property

.gitignore

@@ -0,0 +1 @@
+debug.test

carray.go

@@ -0,0 +1,45 @@
+package cwclient
+
+import "time"
+
+// Carray 每次url请求都会携带一个值. 可以为nil
+type Carray struct {
+	data   interface{} // 携带的数据. 每次请求唯一.
+	hash   string      // 每次请求唯一hash. 标识.
+	expire time.Time   // 过期时间
+}
+
+// GetExpire Get return expire time.Time
+func (undefined *Carray) GetExpire() time.Time {
+	return undefined.expire
+}
+
+// SetExpire Set expire time.Time
+func (undefined *Carray) SetExpire(expire time.Time) {
+	undefined.expire = expire
+}
+
+// GetHash Get return hash string
+func (undefined *Carray) GetHash() string {
+	return undefined.hash
+}
+
+// SetHash Set hash string
+func (undefined *Carray) SetHash(hash string) {
+	undefined.hash = hash
+}
+
+// GetData Get return data interface{}
+func (undefined *Carray) GetData() interface{} {
+	return undefined.data
+}
+
+// SetData Set data interface{}
+func (undefined *Carray) SetData(data interface{}) {
+	undefined.data = data
+}
+
+// Timeup 判断是否过期,超时
+func (undefined *Carray) Timeup() bool {
+	return time.Now().Before(undefined.expire)
+}

client.go

@@ -1,4 +1,4 @@
-package cplient
+package cwclient
 
 import (
 	"context"
@@ -8,24 +8,40 @@ import (
 	"net"
 	"net/http"
 	"net/url"
+	"os"
+	"strconv"
 	"sync"
+	"time"
 
 	"github.com/gin-gonic/gin"
 	"github.com/google/uuid"
 )
 
+func init() {
+	log.SetFlags(log.Llongfile | log.LstdFlags)
+}
+
+// CallbackContext Callback上下文
+type CallbackContext struct {
+	TaskID  string
+	Content string
+	Error   error
+	Carry   interface{} // 传递的参数.
+}
+
 // Callback 发送代理连接获取内容后的回调函数
 type Callback struct {
 	label string
 	hash  string
-	Do    func(tid, content string)
+	Do    func(cxt *CallbackContext)
 }
 
 // Client 客户端
 type Client struct {
 	chromeProxyAddr string
 
-	register sync.Map
+	carrayCache sync.Map
+	register    sync.Map
 
 	host string
 	port string
@@ -38,7 +54,35 @@ type Client struct {
 // Label 区分不同任务类型
 type Label struct {
 	label string
-	cli   *Client
+
+	conditionJS string
+	retry       string
+	waitcapture string
+	configlock  sync.Mutex
+
+	cli *Client
+}
+
+// GetWaitime Get return waitime int
+func (l *Label) GetWaitime() int {
+	r, _ := strconv.Atoi(l.waitcapture)
+	return r
+}
+
+// SetWaitime Set waitime int
+func (l *Label) SetWaitime(waitime int) {
+	l.waitcapture = strconv.Itoa(waitime)
+}
+
+// GetRetry Get return retry int
+func (l *Label) GetRetry() int {
+	r, _ := strconv.Atoi(l.retry)
+	return r
+}
+
+// SetRetry Set retry int
+func (l *Label) SetRetry(retry int) {
+	l.retry = strconv.Itoa(retry)
 }
 
 // GetHash 根据label获取hash路径
@@ -57,9 +101,32 @@ func (l *Label) GetLabel(hash string) string {
 	return ""
 }
 
-// Open 缓存了Label值. 每次调用少了label传参
-func (l *Label) Open(urlstr string) (bodyRes string, ok bool) {
-	return l.cli.Open(l.label, urlstr)
+// Open 缓存了Label值. 每次调用少了label传参. carray每次都会给一次请求回调传入参数.
+func (l *Label) Open(urlstr string, carray interface{}) (bodyRes string, ok bool) {
+	return l.cli.open(l, urlstr, carray)
+}
+
+// SetContentCondition 设置识别到的内容条件. js代码. 必须是一个函数. 命名可以随意. 返回bool
+func (l *Label) SetContentCondition(jsScript string) {
+	l.configlock.Lock()
+	defer l.configlock.Unlock()
+	l.conditionJS = jsScript
+}
+
+// SetContentConditionFromFile 设置识别到的内容条件. js代码. 从js文件 必须是一个函数. 命名可以随意. 返回bool
+func (l *Label) SetContentConditionFromFile(jsScriptFile string) {
+	f, err := os.Open(jsScriptFile)
+	if err != nil {
+		panic(err)
+	}
+	data, err := ioutil.ReadAll(f)
+	if err != nil {
+		panic(err)
+	}
+	l.configlock.Lock()
+	defer l.configlock.Unlock()
+	// log.Println(string(data))
+	l.conditionJS = string(data)
 }
 
 // GetPort Get return port string. default random.
@@ -83,14 +150,14 @@ func (cli *Client) SetHost(host string) {
 }
 
 // Register 注册基础信息
-func (cli *Client) Register(label string, callback func(tid, content string)) *Label {
+func (cli *Client) Register(label string, callback func(cxt *CallbackContext)) *Label {
 	cb := Callback{Do: callback, hash: uuid.New().String()}
 	if _, ok := cli.register.Load(label); ok {
 		log.Panic("label: ", label, " is exists")
 	}
 	cli.register.Store(label, cb)
 	cli.register.Store(cb.hash, cb)
-	l := &Label{label: label, cli: cli}
+	l := &Label{label: label, cli: cli, waitcapture: "6000", retry: "1"}
 	return l
 }
 
@@ -123,7 +190,26 @@ func (cli *Client) Connect() {
 				callback := f.(Callback)
 				if tid, ok := c.GetPostForm("taskid"); ok {
 					content := c.PostForm("content")
-					callback.Do(tid, content)
+					errorStr := c.PostForm("error")
+					carrayhash := c.PostForm("carrayhash")
+
+					var carray interface{}
+					if icarray, ok := cli.carrayCache.Load(carrayhash); ok {
+						carray = icarray.(*Carray).data
+					}
+
+					var err error = nil
+					if errorStr != "" {
+						err = fmt.Errorf(errorStr)
+					}
+
+					cxt := &CallbackContext{
+						TaskID:  tid,
+						Content: content,
+						Error:   err,
+						Carry:   carray,
+					}
+					callback.Do(cxt)
 				}
 			}
 		})
@@ -155,19 +241,36 @@ func (cli *Client) Disconnect() {
 	}
 }
 
-// Open  请求完url后 调用不同label注册的回调函数
-func (cli *Client) Open(label, urlstr string) (bodyRes string, ok bool) {
+// open  请求完url后 调用不同label注册的回调函数. bodyRes 请求后服务器返回的基础信息. 如果不需要debug一般不需要使用.
+func (cli *Client) open(label *Label, urlstr string, carray interface{}) (bodyRes string, ok bool) {
 	// urlstr = "https://playerduo.com/api/playerDuo-service-v2/rip113?lang=en&deviceType=browser"
 
 	if cli.server == nil {
 		panic("client is not connect. Client.Connect() ? ")
 	}
 
-	if callback, ok := cli.register.Load(label); ok {
+	if callback, ok := cli.register.Load(label.label); ok {
+
 		data := url.Values{}
 		data["url"] = []string{urlstr}
 		data["callback"] = []string{cli.host + ":" + cli.port + "/" + callback.(Callback).hash}
-		data["label"] = []string{label}
+		data["label"] = []string{label.label}
+
+		if carray != nil {
+			carrayhash := uuid.New().String()
+			c := &Carray{hash: carrayhash, data: carray, expire: time.Now().Add(time.Minute * 2)}
+			label.cli.carrayCache.Store(carrayhash, c)
+			data["carrayhash"] = []string{carrayhash}
+		}
+
+		func() {
+			label.configlock.Lock()
+			defer label.configlock.Unlock()
+			data["content_condition"] = []string{label.conditionJS}
+			data["waitcapture"] = []string{label.waitcapture}
+			data["retry"] = []string{label.retry}
+		}()
+
 		resp, err := http.DefaultClient.PostForm(cli.chromeProxyAddr+"/task/put", data)
 		if err != nil {
 			panic(err)
@@ -180,7 +283,7 @@ func (cli *Client) Open(label, urlstr string) (bodyRes string, ok bool) {
 		return string(bodyRes), true
 	}
 
-	log.Printf("label: %s is not exists", label)
+	log.Printf("label: %s is not exists", label.label)
 	return "", false
 }
 

client_test.go

@@ -1,87 +1,36 @@
-package cplient
+package cwclient
 
 import (
-	"fmt"
+	"encoding/json"
 	"log"
 	"net/http"
-	"os"
 	"testing"
-	"time"
-
-	"github.com/tebeka/selenium"
-	"github.com/tebeka/selenium/chrome"
 )
 
 func TestPort(t *testing.T) {
 	cli := New("http://localhost:7123")
-	ltest := cli.Register("test", func(tid, content string) {
-		log.Println(tid, content)
 
+	ltest := cli.Register("test", func(cxt *CallbackContext) {
+		if cxt.Error != nil {
+			log.Println("error:", cxt.Error)
+		} else {
+			log.Println(cxt.TaskID, cxt.Content, cxt.Carry)
+		}
 	})
+
+	ltest.SetContentConditionFromFile("example.js")
 	cli.Connect()
-	log.Println(ltest.Open("https://playerduo.com/api/playerDuo-service-v2/rip113?lang=en&deviceType=browser"))
-	log.Println(ltest.Open("https://playerduo.com/api/playerDuo-service-v2/yanngu?lang=en&deviceType=browser"))
-	log.Println(ltest.Open("https://playerduo.com/api/playerDuo-service-v2/rip113?lang=en&deviceType=browser"))
-	log.Println(ltest.Open("https://playerduo.com/api/playerDuo-service-v2/yanngu?lang=en&deviceType=browser"))
-	http.ListenAndServe(":42311", nil)
+	log.Println(ltest.Open("https://playerduo.com/api/playerDuo-service-v2/rip113?lang=en&deviceType=browser", nil))
+	// log.Println(ltest.Open("https://playerduo.com/api/playerDuo-service-v2/yanngu?lang=en&deviceType=browser", "213"))
+	// log.Println(ltest.Open("https://playerduo.com/api/playerDuo-service-v2/rip113?lang=en&deviceType=browser"))
+	// log.Println(ltest.Open("https://playerduo.com/api/playerDuo-service-v2/yanngu?lang=en&deviceType=browser"))
+	http.ListenAndServe(":4233", nil)
 }
 
 func TestChrome(t *testing.T) {
-	port := 41331
 
-	var err error
-	caps := selenium.Capabilities{"browserName": "chrome"}
-
-	chromecaps := chrome.Capabilities{}
-	// for _, epath := range []string{"../../../crx/myblock.crx", "../../crx/myblock.crx", "./crx/myblock.crx"} {
-	// 	_, err := os.Stat(epath)
-	// 	if err == nil {
-	// 		err := chromecaps.AddExtension(epath)
-	// 		if err != nil {
-	// 			panic(err)
-	// 		}
-	// 		break
-	// 	}
-	// }
-
-	chromecaps.Args = append(chromecaps.Args, "--disable-blink-features=AutomationControlled")
-	chromecaps.Args = append(chromecaps.Args, "user-agent=Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.102 Safari/537.36")
-	if proxy := os.Getenv("chrome_proxy"); proxy != "" {
-		log.Println("proxy-server", proxy)
-		chromecaps.Args = append(chromecaps.Args, "--proxy-server="+proxy)
-	}
-
-	if proxy := os.Getenv("pac_proxy"); proxy != "" {
-		log.Println("--proxy-pac-url=" + proxy)
-		chromecaps.Args = append(chromecaps.Args, "--proxy-pac-url="+proxy)
-	}
-
-	// chromecaps.Args = append(chromecaps.Args, "--proxy-pac-url=http://127.0.0.1:1081/pac")
-	chromecaps.Args = append(chromecaps.Args, "--disk-cache-dir=/tmp/chromedriver-cache")
-	chromecaps.Args = append(chromecaps.Args, "--user-data-dir=/tmp/chromedriver-userdata")
-	chromecaps.Args = append(chromecaps.Args, "--auto-open-devtools-for-tabs")
-
-	chromecaps.Args = append(chromecaps.Args, "--disable-gpu", "--disable-images", "--start-maximized", "--disable-infobars")
-	// chromecaps.Args = append(chromecaps.Args, "--headless")
-	chromecaps.Args = append(chromecaps.Args, "--no-sandbox")
-	chromecaps.Args = append(chromecaps.Args, "--disable-dev-shm-usage", "--mute-audio", "--safebrowsing-disable-auto-update")
-
-	chromecaps.ExcludeSwitches = append(chromecaps.ExcludeSwitches, "enable-automation")
-	caps.AddChrome(chromecaps)
-
-	_, err = selenium.NewChromeDriverService("/usr/bin/chromedriver", port)
-	if err != nil {
-		panic(err)
-	}
-
-	wd, err := selenium.NewRemote(caps, fmt.Sprintf("http://localhost:%d/wd/hub", 9222))
-	if err != nil {
-		panic(err)
-	}
-
-	err = wd.Get("https://www.amazon.com/")
-	if err != nil {
-		log.Println(err)
-	}
-	time.Sleep(time.Second * 10)
+	var a = make(map[string]interface{})
+	a["as"] = 123
+	data, _ := json.Marshal(a["as"])
+	log.Println(string(data), a["as"])
 }

example.js

@@ -0,0 +1,6 @@
+
+condition = function () {
+    var href = window.location.href;
+    var content = document.documentElement.innerHTML;
+    return href.startsWith("https://playerduo.com") && content.startsWith('<head></head><body><pre style="word-wrap: break-word; white-space: pre-wrap;">')
+}

go.sum

@@ -3,8 +3,11 @@ cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT
 cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
 cloud.google.com/go v0.41.0/go.mod h1:OauMR7DV8fzvZIl2qg6rkaIhD/vmgk4iwEw/h6ercmg=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802 h1:1BDTz0u9nC3//pOCMdNH+CiXJVYJh5UQNCOBG7jbELc=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/BurntSushi/xgbutil v0.0.0-20160919175755-f7c97cef3b4e h1:4ZrkT/RzpnROylmoQL57iVUL57wGKTR5O6KpVnbm2tA=
 github.com/BurntSushi/xgbutil v0.0.0-20160919175755-f7c97cef3b4e/go.mod h1:uw9h2sd4WWHOPdJ13MQpwK5qYWKYDumDqxWWIknEQ+k=
+github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
 github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
 github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
@@ -24,6 +27,7 @@ github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD87
 github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
 github.com/go-playground/validator/v10 v10.2.0 h1:KgJ0snyC2R9VXYN2rneOtQcw5aHQB1Vv0sFl1UcHBOY=
 github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
@@ -35,6 +39,7 @@ github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaW
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-github/v27 v27.0.4/go.mod h1:/0Gr8pJ55COkmv+S/yPKCczSkUPIM/LnFyubufRNIS0=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
@@ -93,6 +98,7 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=