TODO: upgrade store method

extractor/nimo_extractor/nimo_extractor_test.go

@@ -1,9 +1,11 @@
 package main
 
 import (
+	"database/sql"
 	"intimate"
 	"log"
 	"testing"
+	"time"
 
 	"github.com/474420502/extractor"
 	"github.com/tebeka/selenium"
@@ -17,7 +19,7 @@ type LiveInfo struct {
 	Followers int64   `exp:"//div[contains(@class,'nimo-rm_followers')]//span[@class='text c2']" mth:"r:ExtractNumber"`
 	Views     int64   `exp:"//div[contains(@class,'nimo-rm_audience')]//span[@class='text c2']" mth:"r:ExtractNumber"`
 	Channel   string  `exp:"//div[contains(@class,'nimo-rm_type')]//span"`
-	Giver     []int64 `exp:"//div[contains(@class,'rank-item-after3')]//span[contains(@class,'nimo-currency__count')]"`
+	Gratuity  []int64 `exp:"//div[contains(@class,'rank-item-after3')]//span[contains(@class,'nimo-currency__count')]"`
 }
 
 func Execute() {
@@ -26,34 +28,59 @@ func Execute() {
 	ps := intimate.NewPerfectShutdown()
 
 	for !ps.IsClose() {
-		// streamer, err := estore.Pop(intimate.PNimo)
+		streamer, err := estore.Pop(intimate.PNimo)
-		// if err != nil {
-		// 	log.Println(err)
-		// 	// estore.UpdateError(streamer, err)
-		// 	continue
-		// }
-		// log.Println(streamer)
-		var pagesource string
-		// wd.Get(streamer.LiveUrl.String)
-		wd.Get("https://www.nimo.tv/live/1253835677")
-
-		waitfor.Default("//div[contains(@class,'nimo-rm_followers')]//span[@class='text c2' and text() != '']", nil)
-		waitfor.Default("//div[contains(@class,'rank-item-top3')]", nil)
-		element, err := wd.FindElement(selenium.ByXPATH, "//div[contains(@class,'rank-item-top3')]")
 		if err != nil {
 			log.Println(err)
+			estore.UpdateError(streamer, err)
+			continue
+		}
+
+		wd.Get(streamer.LiveUrl.String)
+		// wd.Get("https://www.nimo.tv/live/1253835677")
+
+		waitfor.Default("//div[contains(@class,'nimo-rm_followers')]//span[@class='text c2' and text() != '']", nil)
+		waitfor.WaitWithTimeout("//div[contains(@class,'rank-item-top3')]", 7*time.Second, nil)
+		element, err := wd.FindElement(selenium.ByXPATH, "//div[contains(@class,'rank-item-top3')]")
+		if err != nil {
+			log.Println(streamer.Uid, err)
 		} else {
 			err = element.MoveTo(50, 50)
 			element.Click()
 			if err != nil {
-				log.Println(err)
+				log.Println(streamer.Uid, err)
 			}
 		}
 		waitfor.Default("//div[contains(@class,'nimo-rm_audience')]//span[@class='text c2']", nil)
 
+		var pagesource string
 		pagesource, _ = wd.PageSource()
+
 		etor := extractor.ExtractHtmlString(pagesource)
 		li := etor.GetObjectByTag(LiveInfo{}).(*LiveInfo)
-		log.Printf("%#v", li)
+		// log.Printf("%#v", li)
+
+		utime := sql.NullTime{Time: time.Now(), Valid: true}
+
+		clog := &intimate.CollectLog{}
+		clog.Platform = intimate.PNimo
+		clog.Followers = sql.NullInt64{Int64: li.Followers, Valid: true}
+		clog.Views = sql.NullInt64{Int64: li.Views, Valid: true}
+		clog.UpdateTime = utime
+		clog.StreamerUid = streamer.Uid
+
+		var sum int64 = 0
+		for _, v := range li.Gratuity {
+			sum += v
+		}
+		clog.Gratuity = sql.NullInt64{Int64: sum, Valid: true}
+
+		cuid := estore.InsertClog(clog)
+
+		streamer.Channel = sql.NullString{String: li.Channel, Valid: true}
+		streamer.LatestLogUid = cuid
+		streamer.UpdateTime = utime
+		streamer.Operator = 0
+
+		estore.Update(streamer, "channel", streamer.Channel, "latest_log_uid", streamer.LatestLogUid, "update_time", streamer.UpdateTime)
 	}
 }

utils.go

@@ -271,6 +271,10 @@ func NewWaitFor(wd selenium.WebDriver) *WaitFor {
 }
 
 func (wf *WaitFor) Default(xpath string, do func(elements ...selenium.WebElement) bool) error {
+	return wf.WaitWithTimeout(xpath, 15*time.Second, do)
+}
+
+func (wf *WaitFor) WaitWithTimeout(xpath string, timeout time.Duration, do func(elements ...selenium.WebElement) bool) error {
 	return wf.WebDriver.WaitWithTimeout(func(wd selenium.WebDriver) (bool, error) {
 		elements, err := wd.FindElements(selenium.ByXPATH, xpath)
 		if err != nil {
@@ -288,5 +292,5 @@ func (wf *WaitFor) Default(xpath string, do func(elements ...selenium.WebElement
 		}
 		return false, nil
 
-	}, time.Second*15)
+	}, timeout)
 }