finish tag slice

extractor/openrec_extractor/openrec_extractor.go

@@ -87,7 +87,7 @@ func (oe *OpenrecExtractor) Execute() {
 		streamer.UpdateTime = source.UpdateTime
 		streamer.Tags = clog.Tags
 
-		clog.Platform = string(intimate.Popenrec)
+		clog.Platform = intimate.Popenrec
 		clog.UserId = userId
 		clog.UpdateTime = source.UpdateTime
 

extractor/twitcasting_extractor/twitcasting_extractor.go

@@ -0,0 +1,140 @@
+package main
+
+import (
+	"database/sql"
+	"intimate"
+	"log"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/474420502/extractor"
+	"github.com/474420502/requests"
+)
+
+// sstore  源存储实例, 为存储源数据的实现. 表格具体参考sql/intimate_source.sql
+var sstore *intimate.StoreSource = intimate.NewStoreSource(string(intimate.STOpenrec))
+
+// estore 解析存储连接实例
+var estore *intimate.StoreExtractor = intimate.NewStoreExtractor()
+
+type LiveData struct {
+	UserName     string   `exp:"//span[@class='tw-live-author__info-username']" method:"Text"`
+	Follower     string   `exp:"(//span[@class='tw-user-nav-list-count'])[2]" method:"Text"`
+	MaxViews     string   `exp:"//span[@id='max_viewer_count']" method:"Text"`
+	LiveTitle    string   `exp:"//meta[@property='og:title']" method:"AttributeValue,content"`
+	LiveStart    string   `exp:"//span[@id='updatetimer']" method:"AttributeValue,data-started-at"`
+	LiveDuration string   `exp:"//span[@id='updatetimer']" method:"AttributeValue,data-duration"`
+	Tags         []string `exp:"//div[@class='tw-live-author__commandbox--tags']//a[@class='tag  tag-info']"`
+}
+
+func main() {
+
+	ps := intimate.NewPerfectShutdown()
+
+	for !ps.IsClose() {
+
+		streamer, err := estore.Pop(intimate.Ptwitcasting)
+		if err != nil {
+			log.Println(err)
+		}
+
+		ses := requests.NewSession()
+		resp, err := ses.Get("https://twitcasting.tv/kyunenee09").Execute()
+		if err != nil {
+			log.Panic(err)
+		}
+		var ldata *LiveData
+		etor := extractor.ExtractXml(resp.Content())
+		ldata = etor.GetObjectByTag(LiveData{}).(*LiveData)
+
+		ldata.MaxViews = regexp.MustCompile("\\d+").FindString(ldata.MaxViews)
+		ldata.LiveStart = ldata.LiveStart[:len(ldata.LiveStart)-3]
+		ldata.LiveDuration = ldata.LiveDuration[:len(ldata.LiveDuration)-3]
+		// log.Println(etor.GetObjectByTag(LiveData{}))
+		coincount := 0
+
+		for i := 0; ; i++ {
+
+			giverurl := "https://twitcasting.tv/kyunenee09/backers/" + strconv.Itoa(i)
+			resp, err = ses.Get(giverurl).Execute()
+			if err != nil {
+				log.Panic(err)
+			}
+			etor := extractor.ExtractXml(resp.Content())
+			xp, err := etor.XPaths("//td[@class='tw-memorial-table-recent-point']")
+			if err != nil {
+				log.Panic(err)
+			}
+
+			coins := xp.GetTexts()
+
+			for _, cointxt := range coins {
+				scointxt := strings.Split(cointxt, "/")
+				if len(scointxt) == 2 {
+					coin := strings.Trim(scointxt[1], " ")
+					c, err := strconv.Atoi(coin)
+					if err == nil {
+						coincount += c
+					}
+					log.Println(coin, coincount)
+				} else {
+					log.Println("coin error: ", cointxt)
+				}
+			}
+
+			if len(coins) < 20 {
+				break
+			}
+		}
+
+		streamer.Platform = intimate.Ptwitcasting
+		streamer.UpdateTime = sql.NullTime{Time: time.Now(), Valid: true}
+		streamer.UpdateInterval = 60
+		streamer.UserName = sql.NullString{String: ldata.UserName, Valid: true}
+		streamer.Operator = 0
+
+		clog := &intimate.CollectLog{}
+		clog.UserId = streamer.UserId
+		clog.Gratuity = sql.NullInt64{Int64: int64(coincount), Valid: true}
+		clog.Platform = streamer.Platform
+		clog.UpdateTime = streamer.UpdateTime
+		clog.LiveTitle = sql.NullString{String: ldata.LiveTitle, Valid: true}
+		fl, err := strconv.Atoi(ldata.Follower)
+		if err == nil {
+			clog.Followers = sql.NullInt64{Int64: int64(fl), Valid: true}
+		} else {
+			log.Println(err)
+		}
+
+		views, err := strconv.Atoi(ldata.MaxViews)
+		if err == nil {
+			clog.Views = sql.NullInt64{Int64: int64(views), Valid: true}
+		} else {
+			log.Println(err)
+		}
+
+		st, err := strconv.Atoi(ldata.LiveStart)
+		if err == nil {
+			startTime := time.Unix(int64(st), 0)
+			clog.LiveStartTime = sql.NullTime{Time: startTime, Valid: true}
+			dt, err := strconv.Atoi(ldata.LiveDuration)
+			if err == nil {
+
+				endTime := startTime.Add((time.Duration)(dt) * time.Second)
+				clog.LiveEndTime = sql.NullTime{Time: endTime, Valid: true}
+			} else {
+				log.Println(err)
+			}
+
+		} else {
+			log.Println(err)
+		}
+
+		streamer.LatestLogUid = estore.InsertClog(clog)
+		estore.UpdateStreamer(streamer)
+
+		break
+	}
+}

extractor/twitcasting_extractor/twitcasting_extractor_test.go

@@ -0,0 +1,12 @@
+package main
+
+import "testing"
+
+// type LiveData struct {
+// 	UserName string `exp:".//span[@class='tw-live-author__info-username']" method:"Text"`
+// 	Follower string `exp:".//span[@class='tw-user-nav-list-count']" method:"Text"`
+// }
+
+func TestMain(t *testing.T) {
+	main()
+}

extractor/twitch_extractor/tiwtch_extractor.go

@@ -6,6 +6,7 @@ import (
 	"intimate"
 	"log"
 	"regexp"
+	"strings"
 	"time"
 
 	"github.com/tebeka/selenium"
@@ -48,6 +49,7 @@ func main() {
 		var updateUrl map[string]string
 		json.Unmarshal(streamer.UpdateUrl.([]byte), &updateUrl)
 		liveUrl := updateUrl["live"]
+		liveUrl = strings.Replace(liveUrl, "/watchparty", "", -1)
 		log.Println(liveUrl)
 
 		// err = wd.Get("https://www.twitch.tv/zoe_0601" + "/about")
@@ -67,6 +69,13 @@ func main() {
 		time.Sleep(time.Millisecond * 500)
 		err = extractUserName(wd, streamer)
 		if err != nil {
+			_, err = wd.FindElement(selenium.ByXPATH, "//a[@data-a-target='browse-channels-button']")
+			if err == nil {
+				log.Println(streamer.UserId, "may be cancell")
+				streamer.Operator = 5
+				streamer.UpdateTime = sql.NullTime{Time: time.Now(), Valid: true}
+				estore.UpdateStreamer(streamer)
+			}
 			continue
 		}
 		err = extractFollowers(wd, clog)
@@ -94,7 +103,7 @@ func main() {
 		}
 
 		streamer.Platform = intimate.Ptwitch
-		clog.Platform = string(streamer.Platform)
+		clog.Platform = streamer.Platform
 		clog.UpdateTime = sql.NullTime{Time: time.Now(), Valid: true}
 		lastClogId := estore.InsertClog(clog)