feat(contentjson): add judge the json of content

chromeproxy/background/capture.js

@@ -1,32 +1,3 @@
-var href = window.location.href;
-var content = document.documentElement.innerHTML;
-
-try {
-    if(waittime === undefined) {
-        waittime = 6000
-    }
-} catch (error) {
-    waittime = 6000
-}
- 
-
-if (condition == undefined) {
-    condition = function () {
-        return true;
-    };
-}
-
-if (condition()) {
-    Tell(BackgroundMsgType.CONTENT, content);
-} else {
-    setTimeout(function () {
-        if (condition()) {
-            Tell(BackgroundMsgType.CONTENT, content);
-        } else {
-            Tell(BackgroundMsgType.NOTWANT, content);
-        }
-    }, waittime)
-}
  
 function Tell(backgroundType, content) {
     try {
@@ -40,3 +11,30 @@ function Tell(backgroundType, content) {
         window.location.href = Host;
     }
 }
+ 
+try {
+    if (condition == undefined) {
+        condition = function () {
+            return true;
+        };
+    }
+} catch (error) {
+    condition = function () {
+        return true;
+    };
+}
+
+var href = window.location.href;
+var content = "";
+if(document.contentType == "application/json") {
+    content = document.documentElement.innerText;
+} else {
+    content = document.documentElement.innerHTML;
+}
+
+if (condition()) {
+    Tell(BackgroundMsgType.CONTENT, content);
+} else {
+    Tell(BackgroundMsgType.NOTWANT, content);
+}
+

chromeproxy/background/worker.js

@@ -16,16 +16,29 @@ chrome.runtime.onMessage.addListener(function (request, sender, sendResponse) {
       GetTask(sender);
       break;
     case BackgroundMsgType.NOTWANT:
-
       task = task_manager[sender.tab.id];
-      task.data.retry --;
-      if(task.data.retry >= 0) {
-        CaptureContent(task);
+      
+      if(task.data.waitcapture != undefined) {
+        setTimeout(function(){
+          CaptureContentCurrentPage(sender.tab.id);
+        }, task.data.waitcapture);
       } else {
-        ErrorTask(sender, request.content);
+
+        task.data.retry --;
+        if(task.data.retry >= 0) {
+          setTimeout(function(){
+            CaptureContent(task);
+          }, 6000)
+        } else {
+          ErrorTask(sender, request.content);
+        }
       }
 
+ 
       break;
+    case BackgroundMsgType.CLOSETAB:
+      delete task_manager[sender.tab.id] 
+      chrome.tabs.remove(sender.tab.id);
     default:
       break;
   }
@@ -35,7 +48,6 @@ function GetTask(sender) {
   fetch(GetTaskUrl).then(function (response) {
     if (response.ok) {
       response.json().then(function (task) {
-        
         CaptureContent(task);
       })
     } else {
@@ -48,6 +60,12 @@ function GetTask(sender) {
   })
 }
 
+function CaptureContentCurrentPage(tabid) {
+  chrome.tabs.executeScript(tabid, { runAt: "document_end", file: "base.js" }, function(){
+    chrome.tabs.executeScript(tabid, { runAt: "document_end", file: "background/capture.js" });
+  });
+}
+
 function CaptureContent(task) {
   if (task.code == 200) {
 
@@ -58,14 +76,17 @@ function CaptureContent(task) {
     chrome.tabs.create({ url: task.data.url }, function (tab) {
       task_manager[tab.id] = task;
       if (task.data.content_condition) {
-        condition = `${task.data.content_condition}`
+        condition = `${task.data.content_condition}`;
+      } else {
+        condition = undefined;
       }
-      if (condition) {
+
+      if (condition != undefined) {
         chrome.tabs.executeScript(tab.id, { runAt: "document_end", code: condition }, function () {
-          chrome.tabs.executeScript(tab.id, { runAt: "document_end", file: "background/capture.js" });
+          CaptureContentCurrentPage(tab.id)
         });
       } else {
-        chrome.tabs.executeScript(tab.id, { runAt: "document_end", file: "background/capture.js" });
+        CaptureContentCurrentPage(tab.id)
       }
     });
   }

chromeproxy/base.js

@@ -19,6 +19,7 @@ const BackgroundMsgType = {
     CONTENT: 'content',
     GETTASK: 'gettask',
     TIMEOUT: 'timeout',
+    CLOSETAB: 'closetab',
     ERROR: 'error',
     OK: 'ok',
 };

chromeproxy/content/inject.js

@@ -5,7 +5,7 @@ if(href.startsWith(Host)) {
   }, 1);
 } else {
   setTimeout(function(){
-    close();
+    chrome.runtime.sendMessage({ type: BackgroundMsgType.CLOSETAB })
   }, 15000)
 }
 

proxyserver/router.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"log"
 	"net/http"
+	"strconv"
 	"time"
 
 	"github.com/bwmarrin/snowflake"
@@ -85,6 +86,18 @@ func PutTask(c *gin.Context) {
 		data.Store("label", label)
 		data.Store("content_condition", c.PostForm("content_condition"))
 
+		if waitcapture, err := strconv.Atoi(c.PostForm("waitcapture")); err != nil {
+			data.Store("waitcapture", 1)
+		} else {
+			data.Store("waitcapture", waitcapture)
+		}
+
+		if retry, err := strconv.Atoi(c.PostForm("retry")); err != nil {
+			data.Store("retry", 1)
+		} else {
+			data.Store("retry", retry)
+		}
+
 		taskQueue.Push(tid, data)
 		oplog.Write(data)
 		c.JSON(http.StatusOK, Response{Code: 200, Message: "ok", Data: data})
@@ -110,7 +123,6 @@ func ContentTask(c *gin.Context) {
 			task.Store("status", "ready")
 			readyQueue.Push(tid, task) // 进入回调发送队列.TODO: 内容持久化
 			c.JSON(200, Response{Code: 200, Data: task})
-			// log.Println("start callback")
 			if label, ok := task.Load("label"); ok {
 				log.Println(label.(string), tid)
 			}

screenlog.0

@@ -0,0 +1,2 @@
+Cannot exec './proxyserver': 权限不够
+

start.sh

@@ -1,6 +1,8 @@
 DISPLAY=:99
 
-screen -L -dmS google-chrome-web  google-chrome --load-extension=../chromeproxy  --user-data-dir=/tmp/chromeproxy-userdata --ignore-certificate-errors --disable-dev-shm-usage --mute-audio --safebrowsing-disable-auto-update --disable-gpu --no-sandbox --disable-blink-features=AutomationControlled --disable-infobars --allow-running-insecure-content --disable-features=TranslateUI --test-type --no-report-upload --display=$DISPLAY
+rm /tmp/chromeproxy-userdata/ -rf
+
+screen -L -dmS google-chrome-web  google-chrome --load-extension=../chromeproxy  --user-data-dir=/tmp/chromeproxy-userdata --ignore-certificate-errors --disable-dev-shm-usage --mute-audio --safebrowsing-disable-auto-update --disable-gpu --no-sandbox --disable-blink-features=AutomationControlled --disable-infobars --allow-running-insecure-content --disable-features=TranslateUI --test-type --no-report-upload --disable-breakpad --no-first-run --display=$DISPLAY
 sleep 2s
 echo "启动浏览器..."
 screen -L -dmS proxyserver ./proxyserver