From 38d369bb580d18cfc368ede5d0211c5966e76dba Mon Sep 17 00:00:00 2001 From: eson Date: Tue, 24 Nov 2020 15:16:37 +0800 Subject: [PATCH] refactor(gettask): 1.simple gettask --- build.sh | 0 build_crx.sh | 0 chromeproxy/background/capture.js | 32 +++++++ chromeproxy/background/config.js | 23 +++++ chromeproxy/background/worker.js | 152 ++++++++++++++---------------- chromeproxy/base.js | 24 ++--- chromeproxy/content/inject.js | 66 +++---------- chromeproxy/manifest.json | 5 +- proxyserver/callback.go | 23 +++++ proxyserver/router.go | 3 +- 10 files changed, 173 insertions(+), 155 deletions(-) mode change 100644 => 100755 build.sh mode change 100644 => 100755 build_crx.sh create mode 100644 chromeproxy/background/capture.js create mode 100644 chromeproxy/background/config.js diff --git a/build.sh b/build.sh old mode 100644 new mode 100755 diff --git a/build_crx.sh b/build_crx.sh old mode 100644 new mode 100755 diff --git a/chromeproxy/background/capture.js b/chromeproxy/background/capture.js new file mode 100644 index 0000000..fcd98ae --- /dev/null +++ b/chromeproxy/background/capture.js @@ -0,0 +1,32 @@ + +var href = window.location.href; +var content = document.documentElement.innerHTML; + +setTimeout(function(){ + window.close(); +}, 15000) + +if (condition == undefined) { + condition = function () { + return true; + }; +} + +if (condition()) { + Tell(BackgroundMsgType.CONTENT, content); +} else { + Tell(BackgroundMsgType.NOTWANT, content); +} + +function Tell(backgroundType, content) { + try { + if (content == undefined) { + chrome.runtime.sendMessage({ type: backgroundType }); + } else { + chrome.runtime.sendMessage({ type: backgroundType, content: content }); + } + } catch (error) { + console.log(error); + window.location.href = Host; + } +} \ No newline at end of file diff --git a/chromeproxy/background/config.js b/chromeproxy/background/config.js new file mode 100644 index 0000000..2168e14 --- /dev/null +++ b/chromeproxy/background/config.js @@ -0,0 +1,23 @@ +// 配置使用的监听 +chrome.webRequest.onBeforeRequest.addListener(function (details) { + if (details.url.startsWith("http://eson.config")) { + var params = new URL(details.url).searchParams; + UpdateHost(params.get("taskurl")) + chrome.tabs.remove(details.tabId); + chrome.tabs.query({ currentWindow: true }, function (tabs) { + var count = 0 + for (var tab of tabs) { + if (!tab.url.startsWith("chrome://")) { + count++; + chrome.tabs.executeScript(tab.id, { runAt: "document_end", code: `window.location.href = "${Host}"` }) + } else { + chrome.tabs.remove(tab.id); + } + } + for (var i = count; i < WorkerNumber; i++) { + chrome.tabs.create({ url: `${Host}` }); + } + }) + return { cancel: true }; + } + }, { 'urls': [""] }, ['blocking']); \ No newline at end of file diff --git a/chromeproxy/background/worker.js b/chromeproxy/background/worker.js index c398094..0def62c 100644 --- a/chromeproxy/background/worker.js +++ b/chromeproxy/background/worker.js @@ -1,58 +1,81 @@ +console.log("background"); var task_manager = { } +chrome.runtime.onMessage.addListener(function (request, sender, sendResponse) { + sendResponse({}); + switch (request.type) { + case BackgroundMsgType.CONTENT: + FinishTask(sender, request.content); + break; + case BackgroundMsgType.ERROR: + ErrorTask(sender, request.error) + break; + case BackgroundMsgType.GETTASK: + GetTask(sender); + break; + case BackgroundMsgType.NOTWANT: + ErrorTask(sender, request.content); + break; + default: + break; + } +}); + function GetTask(sender) { fetch(GetTaskUrl).then(function (response) { if (response.ok) { response.json().then(function (task) { - if(task.code == 200) { - task_manager[sender.tab.id] = task; - if(task.data.content_condition) { - execCode = `PassContentCondition = ${task.data.content_condition};` - } - chrome.tabs.executeScript(sender.tab.id, {runAt: "document_end", code: `window.location.href = "${task.data.url}";`}, function (result) { - if(execCode) { - console.log(execCode); - chrome.tabs.executeScript(sender.tab.id, {code: execCode}); - } - }) - Tell(sender, InjectMsgType.FETCH); - } else { - Tell(sender, InjectMsgType.NOTASK); - } + CaptureContent(task); }) } else { - // ErrorTask(sender, "error response:" + response.text()) - console.log("error response:",response.text()) + console.log("error response:", response.text()) + ErrorTask(sender, response.text()); } }).catch(function (reason) { console.log(reason); - // GetTask(); + ErrorTask(sender, reason); }) } +function CaptureContent(task) { + if (task.code == 200) { + chrome.tabs.create({ url: task.data.url }, function (tab) { + task_manager[tab.id] = task; + if (task.data.content_condition) { + condition = `${task.data.content_condition}` + } + if (condition) { + chrome.tabs.executeScript(tab.id, { runAt: "document_end", code: condition }, function () { + chrome.tabs.executeScript(tab.id, { runAt: "document_end", file: "background/capture.js" }); + }); + } else { + chrome.tabs.executeScript(tab.id, { runAt: "document_end", file: "background/capture.js" }); + } + }); + } +} + function FinishTask(sender, content) { var task = task_manager[sender.tab.id]; - var formdata = new FormData(); formdata.append("taskid", task.data.taskid); formdata.append("content", content); - // console.log(task.data.taskid); - fetch(FinishTaskUrl, {method: "POST", body: formdata }).then(function (response) { - if (response.ok) { - response.json().then(function (value) { + delete task_manager[sender.tab.id]; + chrome.tabs.remove(sender.tab.id); + fetch(FinishTaskUrl, { method: "POST", body: formdata }).then(function (response) { + if (response.ok) { + response.json().then(function (value) { if (value.code == 200) { GetTask(sender); return } }); } - }).catch(function(error){ - // TODO: 汇报错误 - // ErrorTask(sender, error); + }).catch(function (error) { + ErrorTask(sender, error); console.log(error) - // GetTask(sender); return; }); } @@ -62,60 +85,23 @@ function ErrorTask(sender, error) { var formdata = new FormData(); formdata.append("taskid", task.data.taskid); formdata.append("error", error); - fetch(ErrorTaskUrl, {method: "POST", body: formdata}); -} - -chrome.runtime.onMessage.addListener(function (request, sender, sendResponse) { - sendResponse({}); - switch (request.type) { - case BackgroundMsgType.CONTENT: - // 重新获取任务 - FinishTask(sender, request.content); - break; - case BackgroundMsgType.ERROR: - // 重新获取任务 - ErrorTask(sender, request.error) - GetTask(sender); - break; - case BackgroundMsgType.GETTASK: - GetTask(sender); - break; - case BackgroundMsgType.NOTWANT: - Tell(sender,InjectMsgType.WAIT); - break; - default: - break; - } -}); - -function Tell(sender, jnjectType) { - try { - chrome.tabs.sendMessage(sender.tab.id , {type: jnjectType}) - } catch (error) { - console.log(error); - } -} - -// 配置使用的监听 -chrome.webRequest.onBeforeRequest.addListener(function (details) { - if (details.url.startsWith("http://eson.config")) { - var params = new URL(details.url).searchParams; - UpdateHost(params.get("taskurl")) - chrome.tabs.remove(details.tabId); - chrome.tabs.query({currentWindow: true}, function(tabs) { - var count = 0 - for(var tab of tabs) { - if(!tab.url.startsWith("chrome://") ) { - count++; - chrome.tabs.executeScript(tab.id, {runAt: "document_end", code: `window.location.href = "${Host}"` }) - } else { - chrome.tabs.remove(tab.id); + fetch(ErrorTaskUrl, { method: "POST", body: formdata }, function (response) { + if (response.ok) { + response.json().then(function (value) { + if (value.code == 200) { + GetTask(sender); + return } - } - for(var i = count; i < WorkerNumber;i++) { - chrome.tabs.create({url: `${Host}`}); - } - }) - return { cancel: true }; - } -}, { 'urls': [""] }, ['blocking']); \ No newline at end of file + }); + } + }).then().catch(function (error) { + console.log(error) + return; + }); +} + + + + + + diff --git a/chromeproxy/base.js b/chromeproxy/base.js index 7999258..fa0b23e 100644 --- a/chromeproxy/base.js +++ b/chromeproxy/base.js @@ -5,14 +5,7 @@ var GetTaskUrl = `${Host}/task/get`; var FinishTaskUrl = `${Host}/task/content`; var ErrorTaskUrl = `${Host}/task/error`; -var WorkerNumber = 10; - -var PassContentCondition = undefined; -// 默认条件所有内容都获取 -var DefaultContentCondition = function () { - return true; - // return href.startsWith("https://playerduo.com") && content.startsWith('
')
-}
+var WorkerNumber = 1;
 
 function UpdateHost(host) {
     Host = host 
@@ -30,10 +23,11 @@ const BackgroundMsgType = {
     OK: 'ok',
 };
 
-const InjectMsgType = {
-    FETCH: 'fetch',
-    WAIT: 'wait',
-    ERROR: 'error',
-    NOTASK: 'notask',
-    OK: 'ok',
-};
+// const InjectMsgType = {
+//     FETCH: 'fetch',
+//     WAIT: 'wait',
+//     ERROR: 'error', 
+//     NOTASK: 'notask', // 没有任务
+//     CAPTURE: "capture", // 捕获内容
+//     OK: 'ok',
+// };
diff --git a/chromeproxy/content/inject.js b/chromeproxy/content/inject.js
index 482ebe9..2e56971 100644
--- a/chromeproxy/content/inject.js
+++ b/chromeproxy/content/inject.js
@@ -1,62 +1,18 @@
-
-
-chrome.runtime.onMessage.addListener(function (request) {
-  switch (request.type) {
-    case InjectMsgType.WAIT:
-      setTimeout(function () {
-        GetTask();
-      }, 6000);
-      break;
-    case InjectMsgType.NOTASK:
-      setTimeout(function () {
-        GetTask();
-      }, 4000);
-      break;
-    case InjectMsgType.FETCH:
-      setTimeout(function () {
-        GetTask();
-      }, 6000)
-      break;
-    default:
-      break;
-  };
-});
-
-
 var href = window.location.href;
-var content = document.documentElement.innerHTML;
-if (href.startsWith(Host)) {
-  GetTask();
-} else {
-  var condition = PassContentCondition != undefined ? PassContentCondition : DefaultContentCondition;
-  if (condition()) {
-    console.log(condition, PassContentCondition);
-    Tell(BackgroundMsgType.CONTENT, content);
-  } else {
-    console.log(condition, PassContentCondition);
-    Tell(BackgroundMsgType.NOTWANT, content);
-  }
-}
-
-
-
-function Tell(backgroundType, content) {
-  try {
-    if (content == undefined) {
-      chrome.runtime.sendMessage({ type: backgroundType });
-    } else {
-      chrome.runtime.sendMessage({ type: backgroundType, content: content });
-    }
-  } catch (error) {
-    console.log(error);
-    window.location.href = Host;
-  }
-
+if(href.startsWith(Host)) {
+  setInterval(function(){
+    GetTask();
+  }, 1);
 }
 
 function GetTask() {
-  Tell(BackgroundMsgType.GETTASK);
-}
+  try {
+    chrome.runtime.sendMessage({ type: BackgroundMsgType.GETTASK }, function(){});
+  } catch (error) {
+    console.log(error);
+  }
+};
+
 
 
 
diff --git a/chromeproxy/manifest.json b/chromeproxy/manifest.json
index 2504df8..60785d7 100644
--- a/chromeproxy/manifest.json
+++ b/chromeproxy/manifest.json
@@ -2,10 +2,13 @@
    "background": {
       "scripts": [
          "base.js",
+         "background/config.js",
          "background/worker.js"
       ]
    },
-   "devtools_page": "devtools.html",
+   "web_accessible_resources": [
+      "background/capture.js"
+   ],
    "browser_action": {
       "default_icon": {
          "16": "static/icons/16.png",
diff --git a/proxyserver/callback.go b/proxyserver/callback.go
index 6000404..30f7dca 100644
--- a/proxyserver/callback.go
+++ b/proxyserver/callback.go
@@ -29,3 +29,26 @@ func CallbackServer(task *Task) {
 		}
 	}
 }
+
+// ErrorCallbackServer 错误处理回调客户端
+func ErrorCallbackServer(task *Task) {
+	defer func() {
+		if err := recover(); err != nil {
+			log.Println(err)
+		}
+	}()
+
+	task.lock.Lock()
+	defer task.lock.Unlock()
+	if callback, ok := task.data["callback"]; ok {
+		cburl := callback.(string)
+		// log.Println(cburl)
+		_, err := requests.NewSession().Post(cburl).SetBodyAuto(task.data, requests.TypeFormData).Execute()
+		if err != nil {
+			log.Println(err)
+			task.data["error"] = fmt.Sprintf("callback url error: %s", err)
+		} else {
+			task.data["status"] = "readied"
+		}
+	}
+}
diff --git a/proxyserver/router.go b/proxyserver/router.go
index 3f922d8..48d9007 100644
--- a/proxyserver/router.go
+++ b/proxyserver/router.go
@@ -75,7 +75,6 @@ func PutTask(c *gin.Context) {
 		data.Store("ts", now.UnixNano())
 		data.Store("label", label)
 		data.Store("content_condition", c.PostForm("content_condition"))
-
 		if callback := c.PostForm("callback"); callback != "" {
 			data.Store("callback", callback)
 		}
@@ -165,6 +164,8 @@ func ErrorTask(c *gin.Context) {
 		task.Store("error", errorStr)
 		errorQueue.Push(tid, task) // 进入回调发送队列.TODO: 内容持久化
 		c.JSON(http.StatusOK, Response{Code: 200})
+		// log.Println(errorStr)
+		go ErrorCallbackServer(task)
 		return
 	}
 	c.JSON(http.StatusOK, Response{Code: 404})