1. 添加了block twitch 视频流的插件.

2. 流量减少99% 3. cpu使用率降低50% 4. 速度提高了400%
2020-07-27 19:30:54 +08:00
parent 41d3763b57
commit 1d2f2d14c5
18 changed files with 186 additions and 75 deletions
--- a/tasks/twitch/twitch_task3/task_twitch_test.go
+++ b/tasks/twitch/twitch_task3/task_twitch_test.go
@@ -0,0 +1,114 @@
+package main
+
+import (
+	"encoding/json"
+	"intimate"
+	"log"
+	"os"
+	"os/signal"
+	"regexp"
+	"sync/atomic"
+	"syscall"
+	"testing"
+	"time"
+
+	"github.com/tebeka/selenium"
+)
+
+// sstore  源存储实例, 为存储源数据的实现. 表格具体参考sql/intimate_source.sql
+var sstore *intimate.StoreSource = intimate.NewStoreSource(string(intimate.STTwitch))
+
+// estore 解析存储连接实例
+var estore *intimate.StoreExtractor = intimate.NewStoreExtractor()
+
+func TestCase(t *testing.T) {
+	var loop int32 = 1
+	wd := intimate.GetChromeDriver(3030)
+
+	go func() {
+		signalchan := make(chan os.Signal)
+		signal.Notify(signalchan, syscall.SIGKILL, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGSTOP)
+		log.Println("accept stop command:", <-signalchan)
+		atomic.StoreInt32(&loop, 0)
+	}()
+
+	var lasterr error = nil
+	// var err error
+
+	for atomic.LoadInt32(&loop) > 0 {
+		streamer, err := estore.Pop(intimate.Ptwitch, 0)
+		if streamer == nil || err != nil {
+			if err != lasterr {
+				log.Println(err, lasterr)
+				lasterr = err
+			}
+			time.Sleep(time.Second * 2)
+			continue
+		}
+
+		var updateUrl map[string]string
+		json.Unmarshal(streamer.UpdateUrl.([]byte), &updateUrl)
+		liveUrl := updateUrl["live"]
+		log.Println(liveUrl)
+
+		err = wd.Get("https://www.twitch.tv/zoe_0601" + "/about")
+		if err != nil {
+			log.Println(err)
+			//estore.UpdateError(streamer, err)
+			continue
+		}
+
+		time.Sleep(time.Millisecond * 500)
+		wd.WaitWithTimeout(func(web selenium.WebDriver) (bool, error) {
+			_, err = web.FindElement(selenium.ByXPATH, "//a[@class='tw-interactive']//h1/text()")
+			if err != nil {
+				return false, err
+			}
+			return true, err
+		}, 12)
+
+		label, err := wd.FindElement(selenium.ByXPATH, "//a[@class='tw-interactive']//h1")
+		if err != nil {
+			log.Println(err)
+			//estore.UpdateError(streamer, err)
+			continue
+		}
+		log.Println(label.Text())
+
+		wd.WaitWithTimeout(func(web selenium.WebDriver) (bool, error) {
+			followers, err := web.FindElement(selenium.ByXPATH, "//div[@data-a-target='about-panel']//div[@class='tw-align-center']/text()")
+			if err != nil {
+				return false, err
+			}
+			ft, err := followers.Text()
+			log.Println(ft)
+			if err != nil || ft != "" {
+				return false, err
+			}
+			return true, nil
+		}, 12)
+
+		followers, err := wd.FindElement(selenium.ByXPATH, "//div[@data-a-target='about-panel']//div[@class='tw-align-center']")
+		if err != nil {
+			log.Println(err)
+			//estore.UpdateError(streamer, err)
+			continue
+		}
+		fstr, err := followers.Text()
+		if err != nil {
+			log.Println(err)
+			//estore.UpdateError(streamer, err)
+			continue
+		}
+		log.Println(regexp.MustCompile(`[\d,]+`).FindString(fstr))
+		//div[@data-a-target="about-panel"]
+
+		if views, err := wd.FindElement(selenium.ByXPATH, "//a[@data-a-target='home-live-overlay-button']/span"); err == nil {
+			log.Println(views.Text())
+			views.Click()
+		}
+
+		streamer.Operator = 0
+		estore.UpdateOperator(streamer)
+	}
+}