TODO: twitch_task2 fix 错误

extractor/openrec_extractor/openrec_extractor.go

@@ -9,12 +9,11 @@ import (
 	"time"
 
 	"github.com/474420502/extractor"
+	"github.com/474420502/gcurl"
+	"github.com/474420502/requests"
 	"github.com/tidwall/gjson"
 )
 
-var estore = intimate.NewStoreExtractor()
-var sstore = intimate.NewStoreSource(string(intimate.STOpenrec))
-
 //UserInfo 提取信息的结构体
 type UserInfo struct {
 	UserName  string `exp:"//p[ contains(@class, 'c-global__user__profile__list__name__text')]"`
@@ -34,135 +33,230 @@ type UserLive struct {
 func Execute() {
 
 	ps := intimate.NewPerfectShutdown()
+	ses := requests.NewSession()
+	squeue := intimate.TStreamer.Queue(intimate.Streamer{}, intimate.ConditionDefault(intimate.Popenrec))
 
 	var lasterr error = nil
 	for !ps.IsClose() {
-		var err error
+		istreamer, err := squeue.Pop()
 
-		source, err := sstore.Pop(intimate.TOpenrecUser, 0)
-		if err != nil {
+		// streamer, err := estore.Pop(intimate.Popenrec) //队列里弹出一个streamer行. 进行解析
+
+		if istreamer == nil || err != nil {
 			if err != lasterr {
 				log.Println(err, lasterr)
 				lasterr = err
 			}
-			time.Sleep(time.Second * 5)
+			time.Sleep(time.Second * 2)
 			continue
 		}
-		lasterr = nil
+		streamer := istreamer.(*intimate.Streamer)
 
-		sdata := source.Ext.([]byte)
-		datamap := gjson.ParseBytes(sdata).Map()
+		userId := *streamer.UserId
 
-		source.Operator = int32(intimate.OperatorError)
-		userId := datamap["var_user_id"].String()
+		var updateUrl map[string]string
+		err = json.Unmarshal(streamer.UpdateUrl.([]byte), &updateUrl) // 反序列化update_url, 里面存了需要采集的url
+		if err != nil {
+			log.Println(err)
+			continue
+		}
+		// Check Userid
 
-		streamer := &intimate.Streamer{}
-		streamer.UserId = userId
-		// streamer.Platform = intimate.Popenrec 不需要更新字段
+		userUrl := updateUrl["user"]
+		log.Println(userUrl)
+		tp := ses.Get(userUrl) // 获取user url页面数据
+		resp, err := tp.Execute()
+		streamer.UpdateTime = &sql.NullTime{Time: time.Now(), Valid: true}
 
-		htmlUser := datamap["html_user"]
-
-		userEtor := extractor.ExtractHtmlString(htmlUser.String())
-		ui, ok1 := userEtor.GetObjectByTag(UserInfo{}).(*UserInfo)
-
-		htmlLive := datamap["html_live"]
-
-		liveEtor := extractor.ExtractHtmlString(htmlLive.String())
-		ul, ok2 := liveEtor.GetObjectByTag(UserLive{}).(*UserLive)
-
-		jsonSupporters := datamap["json_supporters"]
-		clog := &intimate.CollectLog{}
-
-		if ok1 {
-			clog.Followers = sql.NullInt64{Int64: ui.Followers, Valid: true}
-			clog.Views = sql.NullInt64{Int64: ui.Views, Valid: true}
-			if ui.Views != 0 {
-				clog.IsLiveStreaming = true
-			}
-			streamer.UserName = sql.NullString{String: ui.UserName, Valid: true}
-
-			giverjson := jsonSupporters
-			var givers []interface{}
-			var gratuity int64 = 0
-
-			for _, v := range giverjson.Array() {
-				giverSource := gjson.Parse(v.String())
-				for _, item := range giverSource.Get("data.items").Array() {
-					givers = append(givers, item.Map())
-					gratuity += item.Get("total_yells").Int()
-				}
-			}
-
-			giversbytes, err := json.Marshal(givers)
-			if err != nil {
-				log.Println(err)
-				clog.ErrorMsg = sql.NullString{String: err.Error(), Valid: true}
-			} else {
-				clog.Giver = giversbytes
-			}
-
-			clog.Gratuity = sql.NullInt64{Int64: gratuity, Valid: true}
-		} else {
-			log.Println("UserInfo may be not exists")
-			estore.UpdateError(streamer, errors.New("UserInfo may be not exists"))
+		if err != nil {
+			log.Println(err)
+			intimate.TStreamer.UpdateError(streamer, err)
 			continue
 		}
 
-		//log.Println(ul)
-		if ok2 {
-			clog.LiveTitle = sql.NullString{String: ul.Title, Valid: true}
+		cookies := ses.GetCookies(tp.GetParsedURL())
 
-			startTime, err := time.ParseInLocation("2006-01-02T15:04:05Z07:00", ul.LiveStartTime, time.Local)
-			if err != nil {
-				log.Println(err)
-			} else {
-				clog.LiveStartTime = sql.NullTime{Time: startTime.Local(), Valid: true}
-				duration, err := intimate.ParseDuration(ul.LiveEndTime)
-				if err != nil {
-					log.Println(err)
-				} else {
-					endTime := startTime.Add(duration)
-					clog.LiveStartTime = sql.NullTime{Time: endTime.Local(), Valid: true}
+		scurl := updateUrl["supporters"] //获取打赏者的数据
+		curl := gcurl.Parse(scurl)
+		supportersSession := curl.CreateSession()
+
+		temporary := curl.CreateTemporary(supportersSession)
+		supportersSession.SetCookies(temporary.GetParsedURL(), cookies)
+		var supporters []string
+		for { // supporters 数据需要登录信息. 下面为赋值 supporters链接获取的uid token random码
+
+			supportersQuery := temporary.GetQuery()
+
+			for _, cookie := range cookies {
+				if cookie.Name == "uuid" {
+					supportersQuery.Set("Uuid", cookie.Value)
+					continue
+				}
+
+				if cookie.Name == "token" {
+					supportersQuery.Set("Token", cookie.Value)
+					continue
+				}
+
+				if cookie.Name == "random" {
+					supportersQuery.Set("Random", cookie.Value)
+					continue
 				}
 			}
 
-			if tags, err := json.Marshal(ul.Tags); err == nil {
-				clog.Tags = tags
-			} else {
-				log.Println("json error", ul.Tags, clog.Tags)
+			supportersQuery.Set("identify_id", userId)
+			temporary.SetQuery(supportersQuery)
+
+			resp, err := temporary.Execute()
+			if err != nil {
+				log.Println(err)
 			}
+			supporterjson := gjson.ParseBytes(resp.Content())
+			supporterdata := supporterjson.Get("data") //解析supporters获取的json数据
+			if supporterdata.Type == gjson.Null {
+				break
+			}
+			supporters = append(supporters, string(resp.Content()))
+
+			temporary.QueryParam("page_number").IntAdd(1)
 		}
 
-		streamer.Uid = source.StreamerId.Int64
-		streamer.UpdateTime = source.UpdateTime
-		if clog.Tags != nil {
-			streamer.Tags = clog.Tags
+		// cookies := cxt.Session().GetCookies(wf.GetParsedURL())
+		// ext := make(map[string]interface{})
+
+		jsonSupporters := supporters
+		htmlUser := string(resp.Content())
+
+		liveUrl := updateUrl["live"]
+		tp = ses.Get(liveUrl)
+		resp, err = tp.Execute()
+		if err != nil {
+			log.Println(err)
+			intimate.TStreamer.UpdateError(streamer, err)
+			continue
 		}
-		clog.Platform = intimate.Popenrec
-		clog.UserId = userId
-		clog.UpdateTime = source.UpdateTime
-		clog.StreamerUid = streamer.Uid
+		htmlLive := string(resp.Content())
+		// ext["var_user_id"] = userId
 
-		logUid := estore.InsertClog(clog)
-
-		LiveUrl := "https://www.openrec.tv/live/" + userId
-
-		streamer.LiveUrl = sql.NullString{String: LiveUrl, Valid: true}
-		streamer.LatestLogUid = logUid
-		// streamer.Operator = 0
-
-		log.Println(streamer.UserId)
-		estore.Update(streamer,
-			"user_name", streamer.UserName,
-			"user_id", streamer.UserId,
-			"live_url", streamer.LiveUrl,
-			"latest_log_uid", streamer.LatestLogUid,
-			"update_time", streamer.UpdateTime,
-			"tags", streamer.Tags,
-		)
-
-		source.Operator = int32(intimate.OperatorExtractorOK)
-		sstore.UpdateOperator(source)
+		// streamer.Platform = intimate.Popenrec
+		streamer.UpdateInterval = 120
+		streamer.UpdateTime = &sql.NullTime{Time: time.Now(), Valid: true}
+		streamer.Operator = 0
+		Extractor(streamer, userId, htmlUser, htmlLive, jsonSupporters)
 	}
 
 }
+
+func Extractor(streamer *intimate.Streamer, userId string, htmlUser, htmlLive string, jsonSupporters []string) {
+
+	// sdata := source.Ext.([]byte)
+	// datamap := gjson.ParseBytes(sdata).Map()
+
+	// userId := datamap["var_user_id"].String()
+
+	// streamer := &intimate.Streamer{}
+	// streamer.UserId = &userId
+	// streamer.Platform = intimate.Popenrec 不需要更新字段
+
+	// htmlUser := datamap["html_user"]
+
+	userEtor := extractor.ExtractHtmlString(htmlUser)
+	ui, ok1 := userEtor.GetObjectByTag(UserInfo{}).(*UserInfo)
+
+	// htmlLive := datamap["html_live"]
+
+	liveEtor := extractor.ExtractHtmlString(htmlLive)
+	ul, ok2 := liveEtor.GetObjectByTag(UserLive{}).(*UserLive)
+
+	// jsonSupporters := datamap["json_supporters"]
+	clog := &intimate.CollectLog{}
+
+	if ok1 {
+		clog.Followers = &sql.NullInt64{Int64: ui.Followers, Valid: true}
+		clog.Views = &sql.NullInt64{Int64: ui.Views, Valid: true}
+		if ui.Views != 0 {
+			clog.IsLiveStreaming = true
+		}
+		streamer.UserName = &sql.NullString{String: ui.UserName, Valid: true}
+
+		// giverjson := jsonSupporters
+		var givers []interface{}
+		var gratuity int64 = 0
+
+		for _, v := range jsonSupporters {
+			giverSource := gjson.Parse(v)
+			for _, item := range giverSource.Get("data.items").Array() {
+				givers = append(givers, item.Map())
+				gratuity += item.Get("total_yells").Int()
+			}
+		}
+
+		giversbytes, err := json.Marshal(givers)
+		if err != nil {
+			log.Println(err)
+			clog.ErrorMsg = &sql.NullString{String: err.Error(), Valid: true}
+		} else {
+			clog.Giver = giversbytes
+		}
+
+		clog.Gratuity = &sql.NullInt64{Int64: gratuity, Valid: true}
+	} else {
+		log.Println("UserInfo may be not exists")
+		intimate.TStreamer.UpdateError(streamer, errors.New("UserInfo may be not exists"))
+		return
+	}
+
+	//log.Println(ul)
+	if ok2 {
+		clog.LiveTitle = &sql.NullString{String: ul.Title, Valid: true}
+
+		startTime, err := time.ParseInLocation("2006-01-02T15:04:05Z07:00", ul.LiveStartTime, time.Local)
+		if err != nil {
+			log.Println(err)
+		} else {
+			clog.LiveStartTime = &sql.NullTime{Time: startTime.Local(), Valid: true}
+			duration, err := intimate.ParseDuration(ul.LiveEndTime)
+			if err != nil {
+				log.Println(err)
+			} else {
+				endTime := startTime.Add(duration)
+				clog.LiveEndTime = &sql.NullTime{Time: endTime.Local(), Valid: true}
+			}
+		}
+
+		if tags, err := json.Marshal(ul.Tags); err == nil {
+			clog.Tags = tags
+		} else {
+			log.Println("json error", ul.Tags, clog.Tags)
+		}
+	}
+
+	// streamer.Uid = source.StreamerId.Int64
+	// streamer.UpdateTime = &source.UpdateTime
+	if clog.Tags != nil {
+		streamer.Tags = clog.Tags
+	}
+	clog.Platform = intimate.Popenrec
+	clog.UserId = userId
+	clog.UpdateTime = &sql.NullTime{Time: time.Now(), Valid: true}
+	clog.StreamerUid = streamer.Uid
+
+	logUid, err := intimate.TClog.InsertRetAutoID(clog)
+	if err != nil {
+		log.Println(err)
+		return
+	}
+
+	LiveUrl := "https://www.openrec.tv/live/" + userId
+
+	streamer.LiveUrl = &sql.NullString{String: LiveUrl, Valid: true}
+	streamer.LatestLogUid = logUid
+	// streamer.Operator = 0
+
+	// log.Println(*streamer.UserId)
+	intimate.TStreamer.Update(streamer)
+
+	// source.Operator = int32(intimate.OperatorExtractorOK)
+	// sstore.UpdateOperator(source)
+
+}

extractor/twitcasting_extractor/twitcasting_extractor.go

@@ -5,7 +5,6 @@ import (
 	"encoding/json"
 	"intimate"
 	"log"
-	"os"
 	"strconv"
 	"strings"
 	"time"
@@ -34,26 +33,35 @@ func main() {
 
 	ps := intimate.NewPerfectShutdown()
 	ses := requests.NewSession()
+	streamerQueue := intimate.TStreamer.Queue(intimate.Streamer{}, intimate.ConditionDefault(intimate.Ptwitcasting))
 
 	for !ps.IsClose() {
 
-		streamer, err := estore.Pop(intimate.Ptwitcasting)
+		// streamer, err := estore.Pop(intimate.Ptwitcasting)
+		isteamer, err := streamerQueue.Pop()
 		if err != nil {
-			log.Println(err, streamer)
+			log.Println(err, isteamer)
+			continue
 		}
 
-		streamer.LiveUrl = sql.NullString{String: "https://twitcasting.tv/" + streamer.UserId, Valid: true}
+		streamer := isteamer.(*intimate.Streamer)
+		streamer.LiveUrl = &sql.NullString{String: "https://twitcasting.tv/" + *streamer.UserId, Valid: true}
 		resp, err := ses.Get(streamer.LiveUrl.String).Execute()
 		if err != nil {
-			estore.UpdateError(streamer, err)
-			log.Println(err, streamer.UserId)
+			intimate.TStreamer.UpdateError(streamer, err)
+			log.Println(err, *streamer.UserId)
 			continue
 		}
 		var ldata *LiveData
-		f, _ := os.OpenFile("./twistcasting.html", os.O_CREATE|os.O_RDWR|os.O_TRUNC, os.ModePerm)
-		f.Write(resp.Content())
+		// f, _ := os.OpenFile("./twistcasting.html", os.O_CREATE|os.O_RDWR|os.O_TRUNC, os.ModePerm)
+		// f.Write(resp.Content())
 		etor := extractor.ExtractHtml(resp.Content())
-		ldata = etor.GetObjectByTag(LiveData{}).(*LiveData)
+		ildata := etor.GetObjectByTag(LiveData{})
+		if ildata == nil {
+			log.Println(streamer.LiveUrl.String)
+			continue
+		}
+		ldata = ildata.(*LiveData)
 		// ldata.MaxViews = regexp.MustCompile("\\d+").FindString(ldata.MaxViews)
 		coincount := 0
 
@@ -62,14 +70,14 @@ func main() {
 			giverurl := streamer.LiveUrl.String + "/backers/" + strconv.Itoa(i)
 			resp, err = ses.Get(giverurl).Execute()
 			if err != nil {
-				estore.UpdateError(streamer, err)
+				intimate.TStreamer.UpdateError(streamer, err)
 				log.Panic(err)
 			}
 
 			etor := extractor.ExtractHtml(resp.Content())
 			xp, err := etor.XPaths("//td[@class='tw-memorial-table-recent-point']")
 			if err != nil {
-				estore.UpdateError(streamer, err)
+				intimate.TStreamer.UpdateError(streamer, err)
 				log.Panic(err)
 			}
 
@@ -100,20 +108,20 @@ func main() {
 		}
 
 		streamer.Platform = intimate.Ptwitcasting
-		streamer.UpdateTime = sql.NullTime{Time: time.Now(), Valid: true}
-		streamer.UserName = sql.NullString{String: ldata.UserName, Valid: true}
+		streamer.UpdateTime = &sql.NullTime{Time: time.Now(), Valid: true}
+		streamer.UserName = &sql.NullString{String: ldata.UserName, Valid: true}
 		streamer.Operator = 0
 		streamer.Tags = tags
 		// streamer.UpdateInterval = 60
 		clog := &intimate.CollectLog{}
-		clog.UserId = streamer.UserId
-		clog.Gratuity = sql.NullInt64{Int64: int64(coincount), Valid: true}
+		clog.UserId = *streamer.UserId
+		clog.Gratuity = &sql.NullInt64{Int64: int64(coincount), Valid: true}
 		clog.Platform = streamer.Platform
 		clog.UpdateTime = streamer.UpdateTime
-		clog.LiveTitle = sql.NullString{String: ldata.LiveTitle, Valid: true}
+		clog.LiveTitle = &sql.NullString{String: ldata.LiveTitle, Valid: true}
 		clog.Tags = tags
 
-		clog.Followers = sql.NullInt64{Int64: int64(ldata.Follower), Valid: true}
+		clog.Followers = &sql.NullInt64{Int64: int64(ldata.Follower), Valid: true}
 		switch {
 		case ldata.Follower <= 100:
 			streamer.UpdateInterval = 720
@@ -125,12 +133,12 @@ func main() {
 			streamer.UpdateInterval = 120
 		}
 
-		clog.Views = sql.NullInt64{Int64: ldata.MaxViews, Valid: true}
+		clog.Views = &sql.NullInt64{Int64: ldata.MaxViews, Valid: true}
 		if ldata.LiveStart != "" {
 			st, err := time.Parse("Mon, 02 Jan 2006 15:04:05 -0700", ldata.LiveStart)
 			if err == nil {
 				startTime := st
-				clog.LiveStartTime = sql.NullTime{Time: startTime, Valid: true}
+				clog.LiveStartTime = &sql.NullTime{Time: startTime, Valid: true}
 				dt, err := strconv.Atoi(ldata.LiveDuration)
 
 				liveduration := time.Now().Sub(startTime)
@@ -149,7 +157,7 @@ func main() {
 
 				if err == nil {
 					endTime := startTime.Add((time.Duration)(dt) * time.Millisecond)
-					clog.LiveEndTime = sql.NullTime{Time: endTime, Valid: true}
+					clog.LiveEndTime = &sql.NullTime{Time: endTime, Valid: true}
 				} else {
 					log.Println(err, streamer.UserId)
 				}
@@ -158,8 +166,16 @@ func main() {
 			}
 		}
 
-		streamer.LatestLogUid = estore.InsertClog(clog)
-		estore.UpdateStreamer(streamer)
-		log.Println(streamer.UserId)
+		clog.StreamerUid = streamer.Uid
+		uid, err := intimate.TClog.InsertRetAutoID(clog)
+		if err != nil {
+			log.Println(err)
+			continue
+		}
+
+		streamer.LatestLogUid = uid
+		intimate.TStreamer.Update(streamer)
+		// estore.UpdateStreamer(streamer)
+		log.Println(*streamer.UserId)
 	}
 }