feat(contentjson): add judge the json of content

.gitignore

@@ -1,2 +1,3 @@
 *.crx
 *.log
+build

build.sh

@@ -1 +1,6 @@
-google-chrome --pack-extension=./chromeproxy --pack-extension-key=./chromeproxy.pem 
+rm build/ -rf
+mkdir -p ./build 
+cd proxyserver && go build && mv proxyserver ../build/ && cd ..
+cp start.sh build/start.sh
+cp stop.sh build/stop.sh
+cp proxyserver/config.yaml build/config.yaml

build_crx.sh

@@ -0,0 +1,2 @@
+
+google-chrome --pack-extension=./chromeproxy --pack-extension-key=./chromeproxy.pem 

chromeproxy/.vscode/launch.json

@@ -14,6 +14,7 @@
                 "--ignore-certificate-errors",
                 "--disable-dev-shm-usage",
                 "--mute-audio",
+                "--single-process",
                 "--safebrowsing-disable-auto-update", 
                 "--disable-gpu",
                 "--no-sandbox",

chromeproxy/background/capture.js

@@ -0,0 +1,40 @@
+ 
+function Tell(backgroundType, content) {
+    try {
+        if (content == undefined) {
+            chrome.runtime.sendMessage({ type: backgroundType });
+        } else {
+            chrome.runtime.sendMessage({ type: backgroundType, content: content });
+        }
+    } catch (error) {
+        console.log(error);
+        window.location.href = Host;
+    }
+}
+ 
+try {
+    if (condition == undefined) {
+        condition = function () {
+            return true;
+        };
+    }
+} catch (error) {
+    condition = function () {
+        return true;
+    };
+}
+
+var href = window.location.href;
+var content = "";
+if(document.contentType == "application/json") {
+    content = document.documentElement.innerText;
+} else {
+    content = document.documentElement.innerHTML;
+}
+
+if (condition()) {
+    Tell(BackgroundMsgType.CONTENT, content);
+} else {
+    Tell(BackgroundMsgType.NOTWANT, content);
+}
+

chromeproxy/background/config.js

@@ -0,0 +1,23 @@
+// 配置使用的监听
+chrome.webRequest.onBeforeRequest.addListener(function (details) {
+    if (details.url.startsWith("http://eson.config")) {
+      var params = new URL(details.url).searchParams;
+      UpdateHost(params.get("taskurl"))
+      chrome.tabs.remove(details.tabId);
+      chrome.tabs.query({ currentWindow: true }, function (tabs) {
+        var count = 0
+        for (var tab of tabs) {
+          if (!tab.url.startsWith("chrome://")) {
+            count++;
+            chrome.tabs.executeScript(tab.id, { runAt: "document_end", code: `window.location.href = "${Host}"` })
+          } else {
+            chrome.tabs.remove(tab.id);
+          }
+        }
+        for (var i = count; i < WorkerNumber; i++) {
+          chrome.tabs.create({ url: `${Host}` });
+        }
+      })
+      return { cancel: true };
+    }
+  }, { 'urls': ["<all_urls>"] }, ['blocking']);

chromeproxy/background/worker.js

@@ -1,37 +1,106 @@
+console.log("background");
 
 var task_manager = {
 }
 
+chrome.runtime.onMessage.addListener(function (request, sender, sendResponse) {
+  sendResponse({});
+  switch (request.type) {
+    case BackgroundMsgType.CONTENT:
+      FinishTask(sender, request.content);
+      break;
+    case BackgroundMsgType.ERROR:
+      ErrorTask(sender, request.error)
+      break;
+    case BackgroundMsgType.GETTASK:
+      GetTask(sender);
+      break;
+    case BackgroundMsgType.NOTWANT:
+      task = task_manager[sender.tab.id];
+      
+      if(task.data.waitcapture != undefined) {
+        setTimeout(function(){
+          CaptureContentCurrentPage(sender.tab.id);
+        }, task.data.waitcapture);
+      } else {
+
+        task.data.retry --;
+        if(task.data.retry >= 0) {
+          setTimeout(function(){
+            CaptureContent(task);
+          }, 6000)
+        } else {
+          ErrorTask(sender, request.content);
+        }
+      }
+
+ 
+      break;
+    case BackgroundMsgType.CLOSETAB:
+      delete task_manager[sender.tab.id] 
+      chrome.tabs.remove(sender.tab.id);
+    default:
+      break;
+  }
+});
+
 function GetTask(sender) {
   fetch(GetTaskUrl).then(function (response) {
     if (response.ok) {
       response.json().then(function (task) {
-        if(task.code == 200) {
-          task_manager[sender.tab.id] = task;
-          chrome.tabs.executeScript(sender.tab.id, {runAt: "document_end", code: `window.location.href = "${task.data.url}"` }, function (results) {
-          })
-          Tell(sender, InjectMsgType.FETCH);
-        } else {
-          Tell(sender, InjectMsgType.NOTASK);
-        }
+        CaptureContent(task);
       })
     } else {
-      // ErrorTask(sender, "error response:" + response.text())
       console.log("error response:", response.text())
+      ErrorTask(sender, response.text());
     }
   }).catch(function (reason) {
     console.log(reason);
-    // GetTask();
+    ErrorTask(sender, reason);
   })
 }
 
+function CaptureContentCurrentPage(tabid) {
+  chrome.tabs.executeScript(tabid, { runAt: "document_end", file: "base.js" }, function(){
+    chrome.tabs.executeScript(tabid, { runAt: "document_end", file: "background/capture.js" });
+  });
+}
+
+function CaptureContent(task) {
+  if (task.code == 200) {
+
+    if(task.data.retry == undefined) {
+      task.data.retry = 1;
+    }
+
+    chrome.tabs.create({ url: task.data.url }, function (tab) {
+      task_manager[tab.id] = task;
+      if (task.data.content_condition) {
+        condition = `${task.data.content_condition}`;
+      } else {
+        condition = undefined;
+      }
+
+      if (condition != undefined) {
+        chrome.tabs.executeScript(tab.id, { runAt: "document_end", code: condition }, function () {
+          CaptureContentCurrentPage(tab.id)
+        });
+      } else {
+        CaptureContentCurrentPage(tab.id)
+      }
+    });
+  }
+}
+
 function FinishTask(sender, content) {
   var task = task_manager[sender.tab.id];
- 
   var formdata = new FormData();
+
   formdata.append("taskid", task.data.taskid);
   formdata.append("content", content);
-  // console.log(task.data.taskid);
+  
+  delete task_manager[sender.tab.id];
+  chrome.tabs.remove(sender.tab.id);
   fetch(FinishTaskUrl, { method: "POST", body: formdata }).then(function (response) {
     if (response.ok) {
       response.json().then(function (value) {
@@ -42,10 +111,8 @@ function FinishTask(sender, content) {
       });
     }
   }).catch(function (error) {
-    // TODO: 汇报错误
-    // ErrorTask(sender, error);
+    ErrorTask(sender, error);
     console.log(error)
-    // GetTask(sender);
     return;
   });
 }
@@ -55,51 +122,23 @@ function ErrorTask(sender, error) {
   var formdata = new FormData();
   formdata.append("taskid", task.data.taskid);
   formdata.append("error", error);
-  fetch(ErrorTaskUrl, {method: "POST", body:  formdata});
-}
-
-chrome.runtime.onMessage.addListener(function (request, sender, sendResponse) {
-  sendResponse({});
-  switch (request.type) {
-    case BackgroundMsgType.CONTENT:
-      // 重新获取任务
-      FinishTask(sender, request.content);
-      break;
-    case BackgroundMsgType.ERROR:
-      // 重新获取任务
-      ErrorTask(sender, request.error)
+  fetch(ErrorTaskUrl, { method: "POST", body: formdata }, function (response) {
+    if (response.ok) {
+      response.json().then(function (value) {
+        if (value.code == 200) {
           GetTask(sender);
-      break;
-    case BackgroundMsgType.GETTASK:
-      GetTask(sender);
-      break;
-    case BackgroundMsgType.NOTWANT:
-      Tell(sender,InjectMsgType.WAIT);
-      break;
-    default:
-      break;
+          return
         }
       });
-
-function Tell(sender, jnjectType) {
-  try {
-    chrome.tabs.sendMessage(sender.tab.id , {type: jnjectType})
-  } catch (error) {
-    console.log(error);
     }
+  }).then().catch(function (error) {
+    console.log(error)
+    return;
+  });
 }
 
-// 配置使用的监听
-chrome.webRequest.onBeforeRequest.addListener(function (details) {
-  if (details.url.startsWith("http://eson.config")) {
-    var params = new URLSearchParams(details.url)
-    UpdateHost(params.get("taskurl"))
-    chrome.tabs.remove(details.tabId, function () { });
-    chrome.tabs.query({currentWindow: true}, function(tabs) {
-      for( tab in tabs) {
-        console.log(tab); 
-      }
-    })
-    return { cancel: true };
-  }
-}, { 'urls': ["<all_urls>"] }, ['blocking']);
+
+ 
+
+
+

chromeproxy/base.js

@@ -5,6 +5,8 @@ var GetTaskUrl = `${Host}/task/get`;
 var FinishTaskUrl = `${Host}/task/content`;
 var ErrorTaskUrl = `${Host}/task/error`;
 
+var WorkerNumber = 1;
+
 function UpdateHost(host) {
     Host = host 
     GetTaskUrl = `${Host}/task/get`
@@ -17,14 +19,16 @@ const BackgroundMsgType = {
     CONTENT: 'content',
     GETTASK: 'gettask',
     TIMEOUT: 'timeout',
+    CLOSETAB: 'closetab',
     ERROR: 'error',
     OK: 'ok',
 };
 
-const InjectMsgType = {
-    FETCH: 'fetch',
-    WAIT: 'wait',
-    ERROR: 'error',
-    NOTASK: 'notask',
-    OK: 'ok',
-};
+// const InjectMsgType = {
+//     FETCH: 'fetch',
+//     WAIT: 'wait',
+//     ERROR: 'error', 
+//     NOTASK: 'notask', // 没有任务
+//     CAPTURE: "capture", // 捕获内容
+//     OK: 'ok',
+// };

chromeproxy/content/inject.js

@@ -1,47 +1,22 @@
-
- 
-chrome.runtime.onMessage.addListener(function (request) {
-  switch(request.type) {
-    case InjectMsgType.WAIT:
-      setTimeout(function(){
-        GetTask();
-      }, 6000);
-      break;
-    case InjectMsgType.NOTASK:
-      setTimeout(function(){
-        GetTask();
-      }, 4000);
-      break;
-    case InjectMsgType.FETCH:
-      setTimeout(function(){
-        GetTask();
-      }, 6000)
-      break;
-    default:
-      break;
-  };
-});
-
- 
 var href = window.location.href;
-var content = document.documentElement.innerHTML;
-if (href.startsWith("https://playerduo.com") && content.startsWith('<head></head><body><pre style="word-wrap: break-word; white-space: pre-wrap;">')) {
-  Tell(BackgroundMsgType.CONTENT, content);
+if(href.startsWith(Host)) {
+  setInterval(function(){
+    GetTask();
+  }, 1);
 } else {
-  Tell(BackgroundMsgType.NOTWANT, content);
-}
-
-function Tell(backgroundType, content) {
-  if(content == undefined) {
-    chrome.runtime.sendMessage({type: backgroundType});
-  } else {
-    chrome.runtime.sendMessage({type: backgroundType, content: content});
-  }
+  setTimeout(function(){
+    chrome.runtime.sendMessage({ type: BackgroundMsgType.CLOSETAB })
+  }, 15000)
 }
 
 function GetTask() {
-  Tell(BackgroundMsgType.GETTASK);
+  try {
+    chrome.runtime.sendMessage({ type: BackgroundMsgType.GETTASK }, function(){});
+  } catch (error) {
+    console.log(error);
   }
+};
+
 
 
 

chromeproxy/manifest.json

@@ -2,10 +2,13 @@
    "background": {
       "scripts": [
          "base.js",
+         "background/config.js",
          "background/worker.js"
       ]
    },
-   "devtools_page": "devtools.html",
+   "web_accessible_resources": [
+      "background/capture.js"
+   ],
    "browser_action": {
       "default_icon": {
          "16": "static/icons/16.png",

proxyserver/callback.go

@@ -15,6 +15,28 @@ func CallbackServer(task *Task) {
 		}
 	}()
 
+	task.lock.Lock()
+	defer task.lock.Unlock()
+	if callback, ok := task.data["callback"]; ok {
+		cburl := callback.(string)
+		_, err := requests.NewSession().Post(cburl).SetBodyAuto(task.data, requests.TypeFormData).Execute()
+		if err != nil {
+			log.Println(err)
+			task.data["error"] = fmt.Sprintf("callback url error: %s", err)
+		} else {
+			task.data["status"] = "readied"
+		}
+	}
+}
+
+// ErrorCallbackServer 错误处理回调客户端
+func ErrorCallbackServer(task *Task) {
+	defer func() {
+		if err := recover(); err != nil {
+			log.Println(err)
+		}
+	}()
+
 	task.lock.Lock()
 	defer task.lock.Unlock()
 	if callback, ok := task.data["callback"]; ok {

proxyserver/go.mod

@@ -4,7 +4,7 @@ go 1.15
 
 require (
 	github.com/474420502/focus v0.12.0
-	github.com/474420502/requests v1.10.0
+	github.com/474420502/requests v1.10.1
 	github.com/bwmarrin/snowflake v0.3.0
 	github.com/gin-gonic/gin v1.6.3
 	github.com/go-playground/validator/v10 v10.4.1 // indirect

proxyserver/go.sum

@@ -3,6 +3,8 @@ github.com/474420502/focus v0.12.0 h1:+icbmj7IEOefvTegHt5EpcHt6WFbe2miIrceUJx2Ev
 github.com/474420502/focus v0.12.0/go.mod h1:d0PMjtMxFz1a9HIhwyFPkWa+JF+0LgOrEUfd8iZka6s=
 github.com/474420502/requests v1.10.0 h1:Oz7+Nx+1iuJJUjbBH3cVmkDVosF2tyq5d72TMAoQH8I=
 github.com/474420502/requests v1.10.0/go.mod h1:SB8/RIUVWF3AGotuq/mATUwAjXzzlV7iWaBNM3+c06c=
+github.com/474420502/requests v1.10.1 h1:k5qK8ZWPIdrG0EurJuK2hhoYFbJ9GJTe0xvrzESMFLo=
+github.com/474420502/requests v1.10.1/go.mod h1:SB8/RIUVWF3AGotuq/mATUwAjXzzlV7iWaBNM3+c06c=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/Pallinder/go-randomdata v1.1.0/go.mod h1:yHmJgulpD2Nfrm0cR9tI/+oAgRqCQQixsA8HyRZfV9Y=
 github.com/bwmarrin/snowflake v0.3.0 h1:xm67bEhkKh6ij1790JB83OujPR5CzNe8QuQqAgISZN0=

proxyserver/router.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"log"
 	"net/http"
+	"strconv"
 	"time"
 
 	"github.com/bwmarrin/snowflake"
@@ -70,14 +71,33 @@ func PutTask(c *gin.Context) {
 		now := time.Now()
 		tid := snowNode.Generate().Base64()
 		label := c.PostForm("label")
-		data.Store("taskid", tid)
-		data.Store("url", u)
-		data.Store("ts", now.UnixNano())
-		data.Store("label", label)
+
+		if carrayhash := c.PostForm("carrayhash"); carrayhash != "" {
+			data.Store("carrayhash", carrayhash)
+		}
 
 		if callback := c.PostForm("callback"); callback != "" {
 			data.Store("callback", callback)
 		}
+
+		data.Store("taskid", tid)
+		data.Store("url", u)
+		data.Store("ts", now.UnixNano())
+		data.Store("label", label)
+		data.Store("content_condition", c.PostForm("content_condition"))
+
+		if waitcapture, err := strconv.Atoi(c.PostForm("waitcapture")); err != nil {
+			data.Store("waitcapture", 1)
+		} else {
+			data.Store("waitcapture", waitcapture)
+		}
+
+		if retry, err := strconv.Atoi(c.PostForm("retry")); err != nil {
+			data.Store("retry", 1)
+		} else {
+			data.Store("retry", retry)
+		}
+
 		taskQueue.Push(tid, data)
 		oplog.Write(data)
 		c.JSON(http.StatusOK, Response{Code: 200, Message: "ok", Data: data})
@@ -103,7 +123,6 @@ func ContentTask(c *gin.Context) {
 			task.Store("status", "ready")
 			readyQueue.Push(tid, task) // 进入回调发送队列.TODO: 内容持久化
 			c.JSON(200, Response{Code: 200, Data: task})
-			// log.Println("start callback")
 			if label, ok := task.Load("label"); ok {
 				log.Println(label.(string), tid)
 			}
@@ -164,6 +183,8 @@ func ErrorTask(c *gin.Context) {
 		task.Store("error", errorStr)
 		errorQueue.Push(tid, task) // 进入回调发送队列.TODO: 内容持久化
 		c.JSON(http.StatusOK, Response{Code: 200})
+		// log.Println(errorStr)
+		go ErrorCallbackServer(task)
 		return
 	}
 	c.JSON(http.StatusOK, Response{Code: 404})

screenlog.0

@@ -0,0 +1,2 @@
+Cannot exec './proxyserver': 权限不够
+

start.sh

@@ -0,0 +1,9 @@
+DISPLAY=:99
+
+rm /tmp/chromeproxy-userdata/ -rf
+
+screen -L -dmS google-chrome-web  google-chrome --load-extension=../chromeproxy  --user-data-dir=/tmp/chromeproxy-userdata --ignore-certificate-errors --disable-dev-shm-usage --mute-audio --safebrowsing-disable-auto-update --disable-gpu --no-sandbox --disable-blink-features=AutomationControlled --disable-infobars --allow-running-insecure-content --disable-features=TranslateUI --test-type --no-report-upload --disable-breakpad --no-first-run --display=$DISPLAY
+sleep 2s
+echo "启动浏览器..."
+screen -L -dmS proxyserver ./proxyserver
+echo "启动代理服务器..."

stop.sh

@@ -0,0 +1,4 @@
+for quitlist in google-chrome-web proxyserver
+do
+    screen -S $quitlist -X quit 
+done