TODO: streamerlist

extractor/nimo_extractor/nimo_extractor.go

@@ -28,10 +28,10 @@ type LiveInfo struct {
 }
 
 func Execute() {
-	wd := intimate.GetChromeDriver()
+	adriver := intimate.GetChromeDriver()
 	count := 0
 	countlimit := 200
-
+	wd := adriver.Webdriver
 	waitfor := intimate.NewWaitFor(wd)
 	ps := intimate.NewPerfectShutdown()
 
@@ -118,9 +118,8 @@ func Execute() {
 		count++
 		if count >= countlimit {
 			count = 0
-			wd.Close()
-			wd.Quit()
-			wd = intimate.GetChromeDriver()
+			adriver.Close()
+			adriver = intimate.GetChromeDriver()
 		}
 	}
 }

extractor/twitch_extractor/tiwtch_extractor.go

@@ -7,7 +7,6 @@ import (
 	"intimate"
 	"log"
 	"regexp"
-	"strings"
 	"time"
 
 	"github.com/tebeka/selenium"
@@ -23,15 +22,13 @@ func main() {
 
 	adriver := intimate.GetChromeDriver()
 
-	defer func() {
-		adriver.Close()
-	}()
-
 	ps := intimate.NewPerfectShutdown()
 
-	queue := intimate.TStreamerList.Queue(intimate.StreamerList{}, intimate.ConditionDefault(intimate.Ptwitch))
+	slqueue := intimate.TStreamerList.Queue(intimate.StreamerList{}, intimate.ConditionDefault(intimate.Ptwitch))
+	squeue := intimate.TStreamer.Queue(intimate.Streamer{}, intimate.ConditionDefault(intimate.Ptwitch))
+
 	var count = 0
-	var countlimt = 1
+	var countlimt = 200
 	var recreate = time.Now()
 
 	var lasterr error = nil
@@ -40,13 +37,36 @@ func main() {
 
 		wd := adriver.Webdriver
 		// sourceChannel, err := sstore.Pop(intimate.TTwitchChannel)
-		isl, err := queue.Pop()
+		isl, err := slqueue.Pop()
 		if err != nil {
 			if lasterr != err {
 				lasterr = err
 				log.Println(err)
 			}
-			time.Sleep(time.Minute)
+
+			istreamer, err := squeue.Pop()
+			if err != nil {
+				if lasterr != err {
+					lasterr = err
+					log.Println(err)
+					ps.Wait(time.Minute)
+					continue
+				}
+			}
+
+			streamer := istreamer.(*intimate.Streamer)
+			Extractor(wd, streamer)
+			if err = intimate.TStreamer.Update(streamer); err != nil {
+				log.Println(err)
+			}
+
+			count++
+			if count >= countlimt || time.Now().Sub(recreate) >= time.Minute*120 {
+				count = 0
+				adriver.Close()
+				adriver = intimate.GetChromeDriver()
+				recreate = time.Now()
+			}
 			continue
 		}
 
@@ -185,17 +205,14 @@ func main() {
 		count++
 		if count >= countlimt || time.Now().Sub(recreate) >= time.Minute*120 {
 			count = 0
-			countlimt = 3
+			adriver.Close()
 			adriver = intimate.GetChromeDriver()
 			recreate = time.Now()
 		}
 
-		if count >= 2 {
-			break
-		}
-
 	}
 
+	adriver.Close()
 }
 
 func Extractor(wd selenium.WebDriver, streamer *intimate.Streamer) {
@@ -211,16 +228,16 @@ func Extractor(wd selenium.WebDriver, streamer *intimate.Streamer) {
 
 	// var updateUrl map[string]string
 	// json.Unmarshal(streamer.UpdateUrl.([]byte), &updateUrl)
-	liveUrl := streamer.LiveUrl.String
-
-	liveUrl = strings.Replace(liveUrl, "/watchparty", "", -1)
+	liveUrl := "https://www.twitch.tv/" + (*streamer.UserId)
+	// liveUrl = strings.Replace(liveUrl, "/watchparty", "", -1)
 	log.Println(liveUrl)
 
 	// err = wd.Get("https://www.twitch.tv/zoe_0601" + "/about")
 	err := wd.Get(liveUrl + "/about")
 	if err != nil {
-		log.Println(err)
-		intimate.TStreamer.UpdateError(streamer, err)
+		errstr := fmt.Errorf("%s: %s", err.Error(), liveUrl+"/about")
+		log.Println(errstr)
+		intimate.TStreamer.UpdateError(streamer, errstr)
 		time.Sleep(time.Second * 5)
 		return
 	}
@@ -235,10 +252,10 @@ func Extractor(wd selenium.WebDriver, streamer *intimate.Streamer) {
 	if err != nil {
 		_, err = wd.FindElement(selenium.ByXPATH, "//a[@data-a-target='browse-channels-button']")
 		if err == nil {
-			log.Println(streamer.UserId, "may be cancell")
+			log.Println(*streamer.UserId, "may be cancell")
 			streamer.Operator = 5
 			streamer.UpdateTime = &sql.NullTime{Time: time.Now(), Valid: true}
-			intimate.TStreamer.UpdateError(streamer, fmt.Errorf(""))
+			intimate.TStreamer.UpdateError(streamer, fmt.Errorf(*streamer.UserId, "may be cancell"))
 		}
 		return
 	}
@@ -276,7 +293,6 @@ func Extractor(wd selenium.WebDriver, streamer *intimate.Streamer) {
 		return
 	}
 
-	streamer.Operator = 10
 	streamer.LatestLogUid = lastClogId
 	if clog.Tags != nil {
 		streamer.Tags = clog.Tags