stable

feat(contentjson): add judge the json of content
feat(config): add retry and waitcapture property
2020-11-27 15:57:24 +08:00 · 2020-11-25 17:44:52 +08:00 · 2020-11-25 16:34:15 +08:00
14 changed files with 161 additions and 71 deletions
--- a/chromeproxy/background/capture.js
+++ b/chromeproxy/background/capture.js
@@ -1,32 +1,3 @@
-var href = window.location.href;
-var content = document.documentElement.innerHTML;
-
-try {
-    if(waittime === undefined) {
-        waittime = 6000
-    }
-} catch (error) {
-    waittime = 6000
-}
- 
-
-if (condition == undefined) {
-    condition = function () {
-        return true;
-    };
-}
-
-if (condition()) {
-    Tell(BackgroundMsgType.CONTENT, content);
-} else {
-    setTimeout(function () {
-        if (condition()) {
-            Tell(BackgroundMsgType.CONTENT, content);
-        } else {
-            Tell(BackgroundMsgType.NOTWANT, content);
-        }
-    }, waittime)
-}
 
 function Tell(backgroundType, content) {
    try {
@@ -40,3 +11,30 @@ function Tell(backgroundType, content) {
        window.location.href = Host;
    }
 }
+ 
+try {
+    if (condition == undefined) {
+        condition = function () {
+            return true;
+        };
+    }
+} catch (error) {
+    condition = function () {
+        return true;
+    };
+}
+
+var href = window.location.href;
+var content = "";
+if(document.contentType == "application/json") {
+    content = document.documentElement.innerText;
+} else {
+    content = document.documentElement.innerHTML;
+}
+
+if (condition()) {
+    Tell("content", content);
+} else {
+    Tell("notwant", content);
+}
+
--- a/chromeproxy/background/config.js
+++ b/chromeproxy/background/config.js
@@ -2,7 +2,27 @@
 chrome.webRequest.onBeforeRequest.addListener(function (details) {
    if (details.url.startsWith("http://eson.config")) {
      var params = new URL(details.url).searchParams;
-      UpdateHost(params.get("taskurl"))
+
+      for( key of params.keys()) {
+        switch(key) {
+          case "proxyserver":
+            ProxyServer = params.get(key)
+          break
+        }
+      }
+
+      var taskurl = params.get("taskurl")
+      if(taskurl !== "") {
+        UpdateHost(taskurl)
+        removeConfigTabAndCreateGetTaskTab(details);
+      }
+
+      return { cancel: true };
+    }
+  }, { 'urls': ["<all_urls>"] }, ['blocking']);
+
+
+function removeConfigTabAndCreateGetTaskTab(details) {
  chrome.tabs.remove(details.tabId);
  chrome.tabs.query({ currentWindow: true }, function (tabs) {
    var count = 0
@@ -18,6 +38,4 @@ chrome.webRequest.onBeforeRequest.addListener(function (details) {
      chrome.tabs.create({ url: `${Host}` });
    }
  })
-      return { cancel: true };
 }
-  }, { 'urls': ["<all_urls>"] }, ['blocking']);
--- a/chromeproxy/background/proxy.js
+++ b/chromeproxy/background/proxy.js
@@ -0,0 +1,21 @@
+
+
+var config = {
+    mode: "pac_script",
+    pacScript: {
+        data: 
+        `
+        ProxyServer = ${ProxyServer};
+        function FindProxyForURL(url, host) {
+            if(ProxyServer != undefined) {
+                return ProxyServer;
+            }
+            return 'DIRECT';
+        }`
+    }
+};
+
+
+chrome.proxy.settings.set(
+    { value: config, scope: 'regular' },
+    function () { });
--- a/chromeproxy/background/worker.js
+++ b/chromeproxy/background/worker.js
@@ -16,16 +16,29 @@ chrome.runtime.onMessage.addListener(function (request, sender, sendResponse) {
      GetTask(sender);
      break;
    case BackgroundMsgType.NOTWANT:
-
      task = task_manager[sender.tab.id];
+      
+      if(task.data.waitcapture != undefined) {
+        setTimeout(function(){
+          CaptureContentCurrentPage(sender.tab.id);
+        }, task.data.waitcapture);
+      } else {
+
        task.data.retry --;
        if(task.data.retry >= 0) {
+          setTimeout(function(){
            CaptureContent(task);
+          }, 6000)
        } else {
          ErrorTask(sender, request.content);
        }
+      }
+
 
      break;
+    case BackgroundMsgType.CLOSETAB:
+      delete task_manager[sender.tab.id] 
+      chrome.tabs.remove(sender.tab.id);
    default:
      break;
  }
@@ -35,7 +48,6 @@ function GetTask(sender) {
  fetch(GetTaskUrl).then(function (response) {
    if (response.ok) {
      response.json().then(function (task) {
-        
        CaptureContent(task);
      })
    } else {
@@ -48,6 +60,10 @@ function GetTask(sender) {
  })
 }

+function CaptureContentCurrentPage(tabid) {
+  chrome.tabs.executeScript(tabid, { runAt: "document_end", file: "background/capture.js" });
+}
+
 function CaptureContent(task) {
  if (task.code == 200) {

@@ -58,14 +74,17 @@ function CaptureContent(task) {
    chrome.tabs.create({ url: task.data.url }, function (tab) {
      task_manager[tab.id] = task;
      if (task.data.content_condition) {
-        condition = `${task.data.content_condition}`
+        condition = `${task.data.content_condition}`;
+      } else {
+        condition = undefined;
      }
-      if (condition) {
-        chrome.tabs.executeScript(tab.id, { runAt: "document_end", code: condition }, function () {
-          chrome.tabs.executeScript(tab.id, { runAt: "document_end", file: "background/capture.js" });
+
+      if (condition != undefined) {
+        chrome.tabs.executeScript(tab.id, { runAt: "document_start", code: condition }, function () {
+          CaptureContentCurrentPage(tab.id)
        });
      } else {
-        chrome.tabs.executeScript(tab.id, { runAt: "document_end", file: "background/capture.js" });
+        CaptureContentCurrentPage(tab.id)
      }
    });
  }
--- a/chromeproxy/base.js
+++ b/chromeproxy/base.js
@@ -1,17 +1,19 @@
-
 var Host = "http://localhost:7123"

 var GetTaskUrl = `${Host}/task/get`;
 var FinishTaskUrl = `${Host}/task/content`;
 var ErrorTaskUrl = `${Host}/task/error`;
+var ConfigUrl = `${Host}/task/config`

 var WorkerNumber = 1;
+var ProxyServer = undefined; 

 function UpdateHost(host) {
    Host = host 
    GetTaskUrl = `${Host}/task/get`
    FinishTaskUrl = `${Host}/task/content`
    ErrorTaskUrl = `${Host}/task/error`
+    ConfigUrl = `${Host}/task/config`
 }

 const BackgroundMsgType = {
@@ -19,6 +21,7 @@ const BackgroundMsgType = {
    CONTENT: 'content',
    GETTASK: 'gettask',
    TIMEOUT: 'timeout',
+    CLOSETAB: 'closetab',
    ERROR: 'error',
    OK: 'ok',
 };
--- a/chromeproxy/content/inject.js
+++ b/chromeproxy/content/inject.js
@@ -5,7 +5,7 @@ if(href.startsWith(Host)) {
  }, 1);
 } else {
  setTimeout(function(){
-    close();
+    chrome.runtime.sendMessage({ type: BackgroundMsgType.CLOSETAB })
  }, 15000)
 }

--- a/chromeproxy/manifest.json
+++ b/chromeproxy/manifest.json
@@ -39,7 +39,7 @@
   "key": "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAlurgt0Ykv740tjk1ebeiSX6UVVRRhxVvh+FvRMTe9PKzkAKUDEW2IjNUz1swxZA8ILnMvtyamAfiErTOxUFB8+zCADU/CK2YEhqUJo3tmHCg6EP2XJL220ZXjmIeft1AqJV1BmGRLhm8VnH8dJ2EThfcflx4JEdlh0/aLJr6UVjF2hPVX8JLAMTVjEfiC82KDNGgXDADqBm3E/6n+Dx+3KhgIYTXKIMk+qRVaOhjAJLh8a9OrDBwpylP5RDifTAyVVa9UOyoLNqynzC9oLabWUr1ovWAOiivhYknFCXnl5q971iNSFpmjc+ZW+aK+TRjMnJF84IeA170corVG3KgnwIDAQAB",
   "manifest_version": 2,
   "name": "ChromeProxy",
-   "permissions": ["background", "http://*/*", "https://*/*",  "storage","tabs" , "activeTab", "webRequest", "webRequestBlocking", "<all_urls>", "webNavigation"],
+   "permissions": ["background", "http://*/*", "https://*/*", "proxy", "storage","tabs" , "activeTab", "webRequest", "webRequestBlocking", "<all_urls>", "webNavigation"],
   "short_name": "ChromeProxy",
   "version": "0.1.0"
 }
--- a/chromeproxy/pac.js
+++ b/chromeproxy/pac.js
@@ -0,0 +1 @@
+
--- a/proxyserver/config.go
+++ b/proxyserver/config.go
@@ -7,7 +7,7 @@ import (
 	"gopkg.in/yaml.v2"
 )

-var config = &Config{}
+var config = &LocalConfig{}

 func init() {
 	f, err := os.Open("config.yaml")
@@ -21,8 +21,8 @@ func init() {
 	}
 }

-// Config 本地yaml配置对象
-type Config struct {
+// LocalConfig 本地yaml配置对象
+type LocalConfig struct {
 	Server struct {
 		URI string `yaml:"uri"`
 	} `yaml:"server"`
@@ -41,3 +41,8 @@ func initChromeProxy() {
 		panic(err)
 	}
 }
+
+// ChromeConfig chrome的配置, 包括代理
+type ChromeConfig struct {
+	PacScript string
+}
--- a/proxyserver/go.mod
+++ b/proxyserver/go.mod
@@ -3,7 +3,7 @@ module git.nonolive.co/eson.hsm/proxyserver
 go 1.15

 require (
-	github.com/474420502/focus v0.12.0
+	github.com/474420502/focus v0.13.1
 	github.com/474420502/requests v1.10.1
 	github.com/bwmarrin/snowflake v0.3.0
 	github.com/gin-gonic/gin v1.6.3
--- a/proxyserver/go.sum
+++ b/proxyserver/go.sum
@@ -1,6 +1,10 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 github.com/474420502/focus v0.12.0 h1:+icbmj7IEOefvTegHt5EpcHt6WFbe2miIrceUJx2Evo=
 github.com/474420502/focus v0.12.0/go.mod h1:d0PMjtMxFz1a9HIhwyFPkWa+JF+0LgOrEUfd8iZka6s=
+github.com/474420502/focus v0.13.0 h1:68G0txM38EinNitcfpP0752k8tF4JRJ0r1mmdH6FoGc=
+github.com/474420502/focus v0.13.0/go.mod h1:d0PMjtMxFz1a9HIhwyFPkWa+JF+0LgOrEUfd8iZka6s=
+github.com/474420502/focus v0.13.1 h1:HwJCqY60eqRAkwKtIefJib1ofBfcvVh5ZTU2xE8Gmkk=
+github.com/474420502/focus v0.13.1/go.mod h1:SrqNq63qSx53TkvVWOuHTbTSvAQTNO8ZUrufXm0Ncq8=
 github.com/474420502/requests v1.10.0 h1:Oz7+Nx+1iuJJUjbBH3cVmkDVosF2tyq5d72TMAoQH8I=
 github.com/474420502/requests v1.10.0/go.mod h1:SB8/RIUVWF3AGotuq/mATUwAjXzzlV7iWaBNM3+c06c=
 github.com/474420502/requests v1.10.1 h1:k5qK8ZWPIdrG0EurJuK2hhoYFbJ9GJTe0xvrzESMFLo=
@@ -127,6 +131,7 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
--- a/proxyserver/router.go
+++ b/proxyserver/router.go
@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"log"
 	"net/http"
+	"strconv"
 	"time"

 	"github.com/bwmarrin/snowflake"
@@ -41,10 +42,9 @@ func init() {
 	task := engine.Group("/task")
 	task.GET("/get", GetTask)
 	task.POST("/put", PutTask)
-
 	task.POST("/content", ContentTask)
-
 	task.POST("/error", ErrorTask)
+	task.GET("/config", GetConfig)

 	task.GET("/ready", ReadyTask)
 	task.POST("/ack", AckTask)
@@ -85,6 +85,18 @@ func PutTask(c *gin.Context) {
 		data.Store("label", label)
 		data.Store("content_condition", c.PostForm("content_condition"))

+		if waitcapture, err := strconv.Atoi(c.PostForm("waitcapture")); err != nil {
+			data.Store("waitcapture", 1)
+		} else {
+			data.Store("waitcapture", waitcapture)
+		}
+
+		if retry, err := strconv.Atoi(c.PostForm("retry")); err != nil {
+			data.Store("retry", 1)
+		} else {
+			data.Store("retry", retry)
+		}
+
 		taskQueue.Push(tid, data)
 		oplog.Write(data)
 		c.JSON(http.StatusOK, Response{Code: 200, Message: "ok", Data: data})
@@ -110,7 +122,6 @@ func ContentTask(c *gin.Context) {
 			task.Store("status", "ready")
 			readyQueue.Push(tid, task) // 进入回调发送队列.TODO: 内容持久化
 			c.JSON(200, Response{Code: 200, Data: task})
-			// log.Println("start callback")
 			if label, ok := task.Load("label"); ok {
 				log.Println(label.(string), tid)
 			}
@@ -162,6 +173,11 @@ func ReadyTask(c *gin.Context) {
 	c.JSON(http.StatusOK, Response{Code: 404, Message: fmt.Sprintf("taskid: %s is not found", tid)})
 }

+// GetConfig 获取任务配置
+func GetConfig(c *gin.Context) {
+
+}
+
 // ErrorTask 任务错误无法完成
 func ErrorTask(c *gin.Context) {
 	tid := c.PostForm("taskid")
--- a/screenlog.0
+++ b/screenlog.0
@@ -0,0 +1,2 @@
+Cannot exec './proxyserver': 权限不够
+
--- a/start.sh
+++ b/start.sh
@@ -1,6 +1,8 @@
 DISPLAY=:99

-screen -L -dmS google-chrome-web  google-chrome --load-extension=../chromeproxy  --user-data-dir=/tmp/chromeproxy-userdata --ignore-certificate-errors --disable-dev-shm-usage --mute-audio --safebrowsing-disable-auto-update --disable-gpu --no-sandbox --disable-blink-features=AutomationControlled --disable-infobars --allow-running-insecure-content --disable-features=TranslateUI --test-type --no-report-upload --display=$DISPLAY
+rm /tmp/chromeproxy-userdata/ -rf
+
+screen -L -dmS google-chrome-web  google-chrome --load-extension=../chromeproxy  --user-data-dir=/tmp/chromeproxy-userdata --ignore-certificate-errors --disable-dev-shm-usage --mute-audio --safebrowsing-disable-auto-update --disable-gpu --no-sandbox --disable-blink-features=AutomationControlled --disable-infobars --allow-running-insecure-content --disable-features=TranslateUI --test-type --no-report-upload --disable-breakpad --no-first-run --display=$DISPLAY
 sleep 2s
 echo "启动浏览器..."
 screen -L -dmS proxyserver ./proxyserver
Author	SHA1	Message	Date
eson	f330daf242	stable	2020-11-27 15:57:24 +08:00
eson	9e2d9d5abc	feat(contentjson): add judge the json of content	2020-11-25 17:44:52 +08:00
eson	b14615f339	feat(config): add retry and waitcapture property	2020-11-25 16:34:15 +08:00
				`@@ -0,0 +1,2 @@`
				`Cannot exec './proxyserver': 权限不够`