feat(config): add retry and waitcapture property

.gitignore

@@ -0,0 +1 @@
+debug.test

carray.go

@@ -0,0 +1,45 @@
+package cwclient
+
+import "time"
+
+// Carray 每次url请求都会携带一个值. 可以为nil
+type Carray struct {
+	data   interface{} // 携带的数据. 每次请求唯一.
+	hash   string      // 每次请求唯一hash. 标识.
+	expire time.Time   // 过期时间
+}
+
+// GetExpire Get return expire time.Time
+func (undefined *Carray) GetExpire() time.Time {
+	return undefined.expire
+}
+
+// SetExpire Set expire time.Time
+func (undefined *Carray) SetExpire(expire time.Time) {
+	undefined.expire = expire
+}
+
+// GetHash Get return hash string
+func (undefined *Carray) GetHash() string {
+	return undefined.hash
+}
+
+// SetHash Set hash string
+func (undefined *Carray) SetHash(hash string) {
+	undefined.hash = hash
+}
+
+// GetData Get return data interface{}
+func (undefined *Carray) GetData() interface{} {
+	return undefined.data
+}
+
+// SetData Set data interface{}
+func (undefined *Carray) SetData(data interface{}) {
+	undefined.data = data
+}
+
+// Timeup 判断是否过期,超时
+func (undefined *Carray) Timeup() bool {
+	return time.Now().Before(undefined.expire)
+}

client.go

@@ -8,24 +8,40 @@ import (
 	"net"
 	"net/http"
 	"net/url"
+	"os"
+	"strconv"
 	"sync"
+	"time"
 
 	"github.com/gin-gonic/gin"
 	"github.com/google/uuid"
 )
 
+func init() {
+	log.SetFlags(log.Llongfile | log.LstdFlags)
+}
+
+// CallbackContext Callback上下文
+type CallbackContext struct {
+	TaskID  string
+	Content string
+	Error   error
+	Carry   interface{} // 传递的参数.
+}
+
 // Callback 发送代理连接获取内容后的回调函数
 type Callback struct {
 	label string
 	hash  string
-	Do    func(tid, content string)
+	Do    func(cxt *CallbackContext)
 }
 
 // Client 客户端
 type Client struct {
 	chromeProxyAddr string
 
-	register sync.Map
+	carrayCache sync.Map
+	register    sync.Map
 
 	host string
 	port string
@@ -38,7 +54,35 @@ type Client struct {
 // Label 区分不同任务类型
 type Label struct {
 	label string
-	cli   *Client
+
+	conditionJS string
+	retry       string
+	waitcapture string
+	configlock  sync.Mutex
+
+	cli *Client
+}
+
+// GetWaitime Get return waitime int
+func (l *Label) GetWaitime() int {
+	r, _ := strconv.Atoi(l.waitcapture)
+	return r
+}
+
+// SetWaitime Set waitime int
+func (l *Label) SetWaitime(waitime int) {
+	l.waitcapture = strconv.Itoa(waitime)
+}
+
+// GetRetry Get return retry int
+func (l *Label) GetRetry() int {
+	r, _ := strconv.Atoi(l.retry)
+	return r
+}
+
+// SetRetry Set retry int
+func (l *Label) SetRetry(retry int) {
+	l.retry = strconv.Itoa(retry)
 }
 
 // GetHash 根据label获取hash路径
@@ -57,9 +101,32 @@ func (l *Label) GetLabel(hash string) string {
 	return ""
 }
 
-// Open 缓存了Label值. 每次调用少了label传参
-func (l *Label) Open(urlstr string) (bodyRes string, ok bool) {
-	return l.cli.Open(l.label, urlstr)
+// Open 缓存了Label值. 每次调用少了label传参. carray每次都会给一次请求回调传入参数.
+func (l *Label) Open(urlstr string, carray interface{}) (bodyRes string, ok bool) {
+	return l.cli.open(l, urlstr, carray)
+}
+
+// SetContentCondition 设置识别到的内容条件. js代码. 必须是一个函数. 命名可以随意. 返回bool
+func (l *Label) SetContentCondition(jsScript string) {
+	l.configlock.Lock()
+	defer l.configlock.Unlock()
+	l.conditionJS = jsScript
+}
+
+// SetContentConditionFromFile 设置识别到的内容条件. js代码. 从js文件 必须是一个函数. 命名可以随意. 返回bool
+func (l *Label) SetContentConditionFromFile(jsScriptFile string) {
+	f, err := os.Open(jsScriptFile)
+	if err != nil {
+		panic(err)
+	}
+	data, err := ioutil.ReadAll(f)
+	if err != nil {
+		panic(err)
+	}
+	l.configlock.Lock()
+	defer l.configlock.Unlock()
+	// log.Println(string(data))
+	l.conditionJS = string(data)
 }
 
 // GetPort Get return port string. default random.
@@ -83,14 +150,14 @@ func (cli *Client) SetHost(host string) {
 }
 
 // Register 注册基础信息
-func (cli *Client) Register(label string, callback func(tid, content string)) *Label {
+func (cli *Client) Register(label string, callback func(cxt *CallbackContext)) *Label {
 	cb := Callback{Do: callback, hash: uuid.New().String()}
 	if _, ok := cli.register.Load(label); ok {
 		log.Panic("label: ", label, " is exists")
 	}
 	cli.register.Store(label, cb)
 	cli.register.Store(cb.hash, cb)
-	l := &Label{label: label, cli: cli}
+	l := &Label{label: label, cli: cli, waitcapture: "6000", retry: "1"}
 	return l
 }
 
@@ -123,7 +190,26 @@ func (cli *Client) Connect() {
 				callback := f.(Callback)
 				if tid, ok := c.GetPostForm("taskid"); ok {
 					content := c.PostForm("content")
-					callback.Do(tid, content)
+					errorStr := c.PostForm("error")
+					carrayhash := c.PostForm("carrayhash")
+
+					var carray interface{}
+					if icarray, ok := cli.carrayCache.Load(carrayhash); ok {
+						carray = icarray.(*Carray).data
+					}
+
+					var err error = nil
+					if errorStr != "" {
+						err = fmt.Errorf(errorStr)
+					}
+
+					cxt := &CallbackContext{
+						TaskID:  tid,
+						Content: content,
+						Error:   err,
+						Carry:   carray,
+					}
+					callback.Do(cxt)
 				}
 			}
 		})
@@ -155,19 +241,36 @@ func (cli *Client) Disconnect() {
 	}
 }
 
-// Open  请求完url后 调用不同label注册的回调函数
-func (cli *Client) Open(label, urlstr string) (bodyRes string, ok bool) {
+// open  请求完url后 调用不同label注册的回调函数. bodyRes 请求后服务器返回的基础信息. 如果不需要debug一般不需要使用.
+func (cli *Client) open(label *Label, urlstr string, carray interface{}) (bodyRes string, ok bool) {
 	// urlstr = "https://playerduo.com/api/playerDuo-service-v2/rip113?lang=en&deviceType=browser"
 
 	if cli.server == nil {
 		panic("client is not connect. Client.Connect() ? ")
 	}
 
-	if callback, ok := cli.register.Load(label); ok {
+	if callback, ok := cli.register.Load(label.label); ok {
+
 		data := url.Values{}
 		data["url"] = []string{urlstr}
 		data["callback"] = []string{cli.host + ":" + cli.port + "/" + callback.(Callback).hash}
-		data["label"] = []string{label}
+		data["label"] = []string{label.label}
+
+		if carray != nil {
+			carrayhash := uuid.New().String()
+			c := &Carray{hash: carrayhash, data: carray, expire: time.Now().Add(time.Minute * 2)}
+			label.cli.carrayCache.Store(carrayhash, c)
+			data["carrayhash"] = []string{carrayhash}
+		}
+
+		func() {
+			label.configlock.Lock()
+			defer label.configlock.Unlock()
+			data["content_condition"] = []string{label.conditionJS}
+			data["waitcapture"] = []string{label.waitcapture}
+			data["retry"] = []string{label.retry}
+		}()
+
 		resp, err := http.DefaultClient.PostForm(cli.chromeProxyAddr+"/task/put", data)
 		if err != nil {
 			panic(err)
@@ -180,7 +283,7 @@ func (cli *Client) Open(label, urlstr string) (bodyRes string, ok bool) {
 		return string(bodyRes), true
 	}
 
-	log.Printf("label: %s is not exists", label)
+	log.Printf("label: %s is not exists", label.label)
 	return "", false
 }
 

client_test.go

@@ -1,87 +1,36 @@
 package cwclient
 
 import (
-	"fmt"
+	"encoding/json"
 	"log"
 	"net/http"
-	"os"
 	"testing"
-	"time"
-
-	"github.com/tebeka/selenium"
-	"github.com/tebeka/selenium/chrome"
 )
 
 func TestPort(t *testing.T) {
 	cli := New("http://localhost:7123")
-	ltest := cli.Register("test", func(tid, content string) {
-		log.Println(tid, content)
 
+	ltest := cli.Register("test", func(cxt *CallbackContext) {
+		if cxt.Error != nil {
+			log.Println("error:", cxt.Error)
+		} else {
+			log.Println(cxt.TaskID, cxt.Content, cxt.Carry)
+		}
 	})
+
+	ltest.SetContentConditionFromFile("example.js")
 	cli.Connect()
-	log.Println(ltest.Open("https://playerduo.com/api/playerDuo-service-v2/rip113?lang=en&deviceType=browser"))
-	log.Println(ltest.Open("https://playerduo.com/api/playerDuo-service-v2/yanngu?lang=en&deviceType=browser"))
-	log.Println(ltest.Open("https://playerduo.com/api/playerDuo-service-v2/rip113?lang=en&deviceType=browser"))
-	log.Println(ltest.Open("https://playerduo.com/api/playerDuo-service-v2/yanngu?lang=en&deviceType=browser"))
-	http.ListenAndServe(":42311", nil)
+	log.Println(ltest.Open("https://playerduo.com/api/playerDuo-service-v2/rip113?lang=en&deviceType=browser", nil))
+	// log.Println(ltest.Open("https://playerduo.com/api/playerDuo-service-v2/yanngu?lang=en&deviceType=browser", "213"))
+	// log.Println(ltest.Open("https://playerduo.com/api/playerDuo-service-v2/rip113?lang=en&deviceType=browser"))
+	// log.Println(ltest.Open("https://playerduo.com/api/playerDuo-service-v2/yanngu?lang=en&deviceType=browser"))
+	http.ListenAndServe(":4233", nil)
 }
 
 func TestChrome(t *testing.T) {
-	port := 41331
 
-	var err error
-	caps := selenium.Capabilities{"browserName": "chrome"}
-
-	chromecaps := chrome.Capabilities{}
-	// for _, epath := range []string{"../../../crx/myblock.crx", "../../crx/myblock.crx", "./crx/myblock.crx"} {
-	// 	_, err := os.Stat(epath)
-	// 	if err == nil {
-	// 		err := chromecaps.AddExtension(epath)
-	// 		if err != nil {
-	// 			panic(err)
-	// 		}
-	// 		break
-	// 	}
-	// }
-
-	chromecaps.Args = append(chromecaps.Args, "--disable-blink-features=AutomationControlled")
-	chromecaps.Args = append(chromecaps.Args, "user-agent=Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.102 Safari/537.36")
-	if proxy := os.Getenv("chrome_proxy"); proxy != "" {
-		log.Println("proxy-server", proxy)
-		chromecaps.Args = append(chromecaps.Args, "--proxy-server="+proxy)
-	}
-
-	if proxy := os.Getenv("pac_proxy"); proxy != "" {
-		log.Println("--proxy-pac-url=" + proxy)
-		chromecaps.Args = append(chromecaps.Args, "--proxy-pac-url="+proxy)
-	}
-
-	// chromecaps.Args = append(chromecaps.Args, "--proxy-pac-url=http://127.0.0.1:1081/pac")
-	chromecaps.Args = append(chromecaps.Args, "--disk-cache-dir=/tmp/chromedriver-cache")
-	chromecaps.Args = append(chromecaps.Args, "--user-data-dir=/tmp/chromedriver-userdata")
-	chromecaps.Args = append(chromecaps.Args, "--auto-open-devtools-for-tabs")
-
-	chromecaps.Args = append(chromecaps.Args, "--disable-gpu", "--disable-images", "--start-maximized", "--disable-infobars")
-	// chromecaps.Args = append(chromecaps.Args, "--headless")
-	chromecaps.Args = append(chromecaps.Args, "--no-sandbox")
-	chromecaps.Args = append(chromecaps.Args, "--disable-dev-shm-usage", "--mute-audio", "--safebrowsing-disable-auto-update")
-
-	chromecaps.ExcludeSwitches = append(chromecaps.ExcludeSwitches, "enable-automation")
-	caps.AddChrome(chromecaps)
-
-	_, err = selenium.NewChromeDriverService("/usr/bin/chromedriver", port)
-	if err != nil {
-		panic(err)
-	}
-
-	wd, err := selenium.NewRemote(caps, fmt.Sprintf("http://localhost:%d/wd/hub", 9222))
-	if err != nil {
-		panic(err)
-	}
-
-	err = wd.Get("https://www.amazon.com/")
-	if err != nil {
-		log.Println(err)
-	}
-	time.Sleep(time.Second * 10)
+	var a = make(map[string]interface{})
+	a["as"] = 123
+	data, _ := json.Marshal(a["as"])
+	log.Println(string(data), a["as"])
 }

example.js

@@ -0,0 +1,6 @@
+
+condition = function () {
+    var href = window.location.href;
+    var content = document.documentElement.innerHTML;
+    return href.startsWith("https://playerduo.com") && content.startsWith('<head></head><body><pre style="word-wrap: break-word; white-space: pre-wrap;">')
+}