完成twitch 数据提取入库

tasks/twitch/twitch_task1/task_twitch.go

@@ -49,22 +49,51 @@ func (cl *ChannelLink) Execute() {
 	}
 	e.Click()
 
+	var hrefs map[string]bool = make(map[string]bool)
+	var delayerror = 5
+	var samecount = 0
 	for i := 0; i <= 200; i++ {
-		wd.KeyDown(selenium.EndKey)
+		cards, err := wd.FindElements(selenium.ByXPATH, "//span/a[contains(@data-a-target,'card-') and @href]")
-		time.Sleep(time.Second * 2)
-	}
-
-	elements, err := wd.FindElements(selenium.ByXPATH, "//span/a[contains(@data-a-target,'card-') and @href]")
-	if err != nil {
-		panic(err)
-	}
-	// xpath: //article//a[@data-a-target='preview-card-title-link']
-	for _, ele := range elements {
-		href, err := ele.GetAttribute("href")
 		if err != nil {
 			log.Println(err)
+			break
 		}
-		log.Println(href) // TODO: Save href
+
+		if len(cards) == samecount {
+			delayerror--
+			if delayerror <= 0 {
+				break
+			}
+		} else {
+			delayerror = 5
+		}
+
+		for _, card := range cards {
+			href, err := card.GetAttribute("href")
+			if err != nil {
+				log.Println(err)
+			} else {
+				hrefs[href] = true
+			}
+		}
+		samecount = len(cards)
+
+		if len(cards) > 10 {
+			log.Println(len(cards))
+			wd.ExecuteScript(`items = document.evaluate("//div[@data-target='directory-page__card-container']/../self::div[@data-target and @style]", document, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null); 
+		for (var i = 0; i < items.snapshotLength - 10; i++) { item = items.snapshotItem(i); item.remove() ;};`, nil)
+		}
+		time.Sleep(time.Millisecond * 200)
+		wd.KeyDown(selenium.EndKey)
+		time.Sleep(time.Millisecond * 200)
+		wd.KeyUp(selenium.EndKey)
+		time.Sleep(time.Millisecond * 2500)
+
+	}
+
+	for href := range hrefs {
+
+		// TODO: Save href
 		source := &intimate.Source{}
 		source.Source = sql.NullString{String: href, Valid: true}
 		source.Operator = 0
@@ -73,5 +102,6 @@ func (cl *ChannelLink) Execute() {
 		sstore.Insert(source)
 	}
 
+	log.Println("hrefs len:", len(hrefs))
 	sstore.Deduplicate(intimate.TTwitchChannel, "source")
 }

tasks/twitch/twitch_task2/task_twitch.go

@@ -36,6 +36,7 @@ func (cl *UserList) Execute() {
 	defer wd.Close()
 
 	var loop int32 = 1
+	var count = 0
 
 	go func() {
 		signalchan := make(chan os.Signal)
@@ -87,12 +88,7 @@ func (cl *UserList) Execute() {
 			wd.KeyUp(selenium.EndKey)
 			time.Sleep(time.Millisecond * 1500)
 			if len(elements) == liveurls {
-				if liveurls == 0 {
+				delayerror--
-					delayerror -= 1
-				} else {
-					delayerror--
-				}
-
 				if delayerror <= 0 {
 					break
 				}
@@ -140,5 +136,11 @@ func (cl *UserList) Execute() {
 			estore.InsertStreamer(streamer)
 		}
 		log.Println("streamer insert", len(elements))
+		count++
+		if count >= 100 {
+			wd.Close()
+			wd = intimate.GetChromeDriver(3030)
+			count = 0
+		}
 	}
 }

tasks/twitch/twitch_task3/task_twitch_test.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"database/sql"
 	"encoding/json"
 	"intimate"
 	"log"
@@ -51,64 +52,170 @@ func TestCase(t *testing.T) {
 		liveUrl := updateUrl["live"]
 		log.Println(liveUrl)
 
-		err = wd.Get("https://www.twitch.tv/zoe_0601" + "/about")
+		// err = wd.Get("https://www.twitch.tv/zoe_0601" + "/about")
+		err = wd.Get(liveUrl + "/about")
 		if err != nil {
 			log.Println(err)
 			//estore.UpdateError(streamer, err)
 			continue
 		}
 
+		streamer.LiveUrl = sql.NullString{String: liveUrl, Valid: true}
+		clog := &intimate.CollectLog{}
+		clog.UserId = streamer.UserId
+
 		time.Sleep(time.Millisecond * 500)
-		wd.WaitWithTimeout(func(web selenium.WebDriver) (bool, error) {
+		extractUserName(wd, streamer)
-			_, err = web.FindElement(selenium.ByXPATH, "//a[@class='tw-interactive']//h1/text()")
+		extractFollowers(wd, clog)
-			if err != nil {
+		err = extractViews(wd, clog) // views + tags + gratuity
-				return false, err
-			}
-			return true, err
-		}, 12)
 
-		label, err := wd.FindElement(selenium.ByXPATH, "//a[@class='tw-interactive']//h1")
 		if err != nil {
-			log.Println(err)
+			// 不直播时提取礼物 gratuity
-			//estore.UpdateError(streamer, err)
+			wd.WaitWithTimeout(func(web selenium.WebDriver) (bool, error) {
-			continue
+				channelchat, err := wd.FindElement(selenium.ByXPATH, `//a[@data-a-target="channel-home-tab-Chat"]`)
-		}
+				btn, _ := web.FindElement(selenium.ByXPATH, `//button[@data-test-selector="expand-grabber"]`)
-		log.Println(label.Text())
+				if (err == nil && channelchat != nil) || btn != nil {
+					if channelchat != nil {
+						channelchat.Click()
+					}
+					time.Sleep(time.Second)
+					extractGratuity(wd, clog)
+					return true, nil
+				}
+				return false, nil
 
-		wd.WaitWithTimeout(func(web selenium.WebDriver) (bool, error) {
+			}, time.Second*4)
-			followers, err := web.FindElement(selenium.ByXPATH, "//div[@data-a-target='about-panel']//div[@class='tw-align-center']/text()")
-			if err != nil {
-				return false, err
-			}
-			ft, err := followers.Text()
-			log.Println(ft)
-			if err != nil || ft != "" {
-				return false, err
-			}
-			return true, nil
-		}, 12)
-
-		followers, err := wd.FindElement(selenium.ByXPATH, "//div[@data-a-target='about-panel']//div[@class='tw-align-center']")
-		if err != nil {
-			log.Println(err)
-			//estore.UpdateError(streamer, err)
-			continue
-		}
-		fstr, err := followers.Text()
-		if err != nil {
-			log.Println(err)
-			//estore.UpdateError(streamer, err)
-			continue
-		}
-		log.Println(regexp.MustCompile(`[\d,]+`).FindString(fstr))
-		//div[@data-a-target="about-panel"]
-
-		if views, err := wd.FindElement(selenium.ByXPATH, "//a[@data-a-target='home-live-overlay-button']/span"); err == nil {
-			log.Println(views.Text())
-			views.Click()
 		}
 
-		streamer.Operator = 0
+		streamer.Platform = intimate.Ptwitch
-		estore.UpdateOperator(streamer)
+		clog.Platform = string(streamer.Platform)
+		clog.UpdateTime = sql.NullTime{Time: time.Now(), Valid: true}
+		lastClogId := estore.InsertCollectLog(clog)
+
+		streamer.Operator = 100
+		streamer.LatestLogUid = lastClogId
+		estore.UpdateStreamer(streamer)
+
 	}
 }
+
+func extractUserName(wd selenium.WebDriver, streamer *intimate.Streamer) error {
+	return wd.WaitWithTimeout(func(web selenium.WebDriver) (bool, error) {
+		label, err := web.FindElement(selenium.ByXPATH, "//a[@class='tw-interactive']//h1")
+		if err == nil {
+			if ltxt, err := label.Text(); err == nil {
+				log.Println("label:", ltxt)
+				streamer.UserName = sql.NullString{String: ltxt, Valid: true}
+				return true, nil
+			}
+		}
+		return false, err
+	}, 6*time.Second)
+}
+
+func extractFollowers(wd selenium.WebDriver, clog *intimate.CollectLog) error {
+	return wd.WaitWithTimeout(func(web selenium.WebDriver) (bool, error) {
+		efollowers, err := web.FindElement(selenium.ByXPATH, "//div[@data-a-target='about-panel']//div[@class='tw-align-center']")
+		if err != nil {
+			return false, err
+		}
+		followers, err := efollowers.Text()
+		if err != nil || followers == "" {
+			return false, err
+		}
+		followers = regexp.MustCompile(`[\d,]+`).FindString(followers)
+		fint, _ := intimate.ParseNumber(followers)
+		clog.Followers = sql.NullInt64{Int64: int64(fint), Valid: true}
+		log.Println("followers: ", followers, fint)
+		return true, nil
+	}, 6*time.Second)
+}
+
+func extractViews(wd selenium.WebDriver, clog *intimate.CollectLog) error {
+	return wd.WaitWithTimeout(func(web selenium.WebDriver) (bool, error) {
+		views, err := web.FindElement(selenium.ByXPATH, "//a[@data-a-target='home-live-overlay-button']/span")
+		if views != nil {
+			if txt, err := views.Text(); err == nil {
+
+				vint, _ := intimate.ParseNumber(txt)
+				clog.Views = sql.NullInt64{Int64: vint, Valid: true}
+				log.Println("views:", txt)
+				views.Click()
+
+				extractTags(wd, clog)
+				extractTitle(wd, clog)
+				extractGratuity(wd, clog)
+
+				return true, nil
+			}
+		}
+		return false, err
+	}, time.Second*4)
+}
+
+func extractTitle(wd selenium.WebDriver, clog *intimate.CollectLog) error {
+	return wd.WaitWithTimeout(func(web selenium.WebDriver) (bool, error) {
+		title, err := web.FindElement(selenium.ByXPATH, `//h2[@data-a-target='stream-title']`)
+		if err == nil {
+			if txt, err := title.Text(); err == nil {
+				clog.LiveTitle = sql.NullString{String: txt, Valid: true}
+				return true, nil
+			}
+		}
+		return false, err
+	}, time.Second*4)
+}
+
+func extractTags(wd selenium.WebDriver, clog *intimate.CollectLog) error {
+	return wd.WaitWithTimeout(func(web selenium.WebDriver) (bool, error) {
+		tags, err := web.FindElements(selenium.ByXPATH, "//a[@aria-label and @data-a-target and @href]/div[@class and text()]")
+		if len(tags) == 0 {
+			return false, err
+		}
+
+		var stags []string
+		for _, tag := range tags {
+			if txt, err := tag.Text(); err == nil {
+				stags = append(stags, txt)
+			} else {
+				log.Println(err)
+			}
+			log.Println(tag.Text())
+		}
+		if len(stags) > 0 {
+			if tagbuf, err := json.Marshal(stags); err == nil {
+				clog.Tags = tagbuf
+			} else {
+				log.Println(err)
+			}
+		}
+
+		return true, nil
+	}, time.Second*4)
+}
+
+func extractGratuity(wd selenium.WebDriver, clog *intimate.CollectLog) error {
+	return wd.WaitWithTimeout(func(web selenium.WebDriver) (bool, error) {
+		btn, err := web.FindElement(selenium.ByXPATH, `//button[@data-test-selector="expand-grabber"]`)
+		if err == nil {
+			btn.Click()
+			time.Sleep(time.Second)
+			gifcount, err := web.FindElements(selenium.ByXPATH, `//div[@class="sub-gift-count tw-flex"]/p`)
+			if err == nil {
+				var gratuity int64 = 0
+				for _, gc := range gifcount {
+					if gtxt, err := gc.Text(); err == nil {
+						gint, _ := intimate.ParseNumber(gtxt)
+						gratuity += gint
+					} else {
+						log.Println(err)
+					}
+				}
+				clog.Gratuity = sql.NullInt64{Int64: gratuity, Valid: true}
+			}
+			return true, nil
+		}
+
+		return false, err
+	}, time.Second*4)
+}

utils.go

@@ -4,6 +4,8 @@ import (
 	"fmt"
 	"log"
 	"runtime"
+	"strconv"
+	"strings"
 	"time"
 
 	"github.com/tebeka/selenium"
@@ -22,6 +24,12 @@ func init() {
 
 }
 
+// ParseNumber 去逗号解析数字
+func ParseNumber(number string) (int64, error) {
+	number = strings.ReplaceAll(number, ",", "")
+	return strconv.ParseInt(number, 10, 64)
+}
+
 // ParseDuration time to duration eg: 1:40:00 -> time.Duration
 func ParseDuration(dt string) (time.Duration, error) {