intimate/extractor/twitcasting_extractor/twitcasting_extractor.go

package main

import (
	"database/sql"
	"intimate"
	"log"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/474420502/extractor"
	"github.com/474420502/requests"
)

// sstore  源存储实例, 为存储源数据的实现. 表格具体参考sql/intimate_source.sql
var sstore *intimate.StoreSource = intimate.NewStoreSource(string(intimate.STOpenrec))

// estore 解析存储连接实例
var estore *intimate.StoreExtractor = intimate.NewStoreExtractor()

type LiveData struct {
	UserName     string   `exp:"//span[@class='tw-live-author__info-username']" method:"Text"`
	Follower     string   `exp:"(//span[@class='tw-user-nav-list-count'])[2]" method:"Text"`
	MaxViews     string   `exp:"//span[@id='max_viewer_count']" method:"Text"`
	LiveTitle    string   `exp:"//meta[@property='og:title']" method:"AttributeValue,content"`
	LiveStart    string   `exp:"//time[@data-kind='relative']" method:"AttributeValue,datetime"`
	LiveDuration string   `exp:"//span[@id='updatetimer']" method:"AttributeValue,data-duration"`
	Tags         []string `exp:"//div[@class='tw-live-author__commandbox--tags']//a[@class='tag  tag-info']" method:"Text"`
}

func main() {

	ps := intimate.NewPerfectShutdown()
	ses := requests.NewSession()

	for !ps.IsClose() {

		streamer, err := estore.Pop(intimate.Ptwitcasting)
		if err != nil {
			log.Println(err, streamer.UserId)
		}

		streamer.LiveUrl = sql.NullString{String: "https://twitcasting.tv/" + streamer.UserId, Valid: true}
		resp, err := ses.Get(streamer.LiveUrl.String).Execute()
		if err != nil {
			estore.UpdateError(streamer, err)
			log.Println(err, streamer.UserId)
			continue
		}
		var ldata *LiveData
		etor := extractor.ExtractHtml(resp.Content())
		ldata = etor.GetObjectByTag(LiveData{}).(*LiveData)
		ldata.MaxViews = regexp.MustCompile("\\d+").FindString(ldata.MaxViews)
		coincount := 0

		for i := 0; ; i++ {

			giverurl := streamer.LiveUrl.String + "/backers/" + strconv.Itoa(i)
			resp, err = ses.Get(giverurl).Execute()
			if err != nil {
				estore.UpdateError(streamer, err)
				log.Panic(err)
			}

			etor := extractor.ExtractHtml(resp.Content())
			xp, err := etor.XPaths("//td[@class='tw-memorial-table-recent-point']")
			if err != nil {
				estore.UpdateError(streamer, err)
				log.Panic(err)
			}

			coins := xp.GetTexts()
			for _, cointxt := range coins {
				scointxt := strings.Split(cointxt, "/")
				if len(scointxt) == 2 {
					coin := strings.Trim(scointxt[1], " ")
					c, err := strconv.Atoi(coin)
					if err == nil {
						coincount += c
					}
					// log.Println(coin, coincount)
				} else {
					log.Println("coin error: ", cointxt)
				}
			}

			if len(coins) < 20 {
				break
			}
		}

		streamer.Platform = intimate.Ptwitcasting
		streamer.UpdateTime = sql.NullTime{Time: time.Now(), Valid: true}
		streamer.UserName = sql.NullString{String: ldata.UserName, Valid: true}
		streamer.Operator = 10
		// streamer.UpdateInterval = 60
		clog := &intimate.CollectLog{}
		clog.UserId = streamer.UserId
		clog.Gratuity = sql.NullInt64{Int64: int64(coincount), Valid: true}
		clog.Platform = streamer.Platform
		clog.UpdateTime = streamer.UpdateTime
		clog.LiveTitle = sql.NullString{String: ldata.LiveTitle, Valid: true}
		fl, err := intimate.ParseNumberEx(ldata.Follower)
		if err == nil {
			clog.Followers = sql.NullInt64{Int64: int64(fl), Valid: true}
			switch {
			case fl <= 100:
				streamer.UpdateInterval = 360
			case fl <= 1000:
				streamer.UpdateInterval = 240
			case fl <= 100:
				streamer.UpdateInterval = 120
			default:
				streamer.UpdateInterval = 60
			}
		} else {
			log.Println(err)
		}

		views, err := strconv.Atoi(ldata.MaxViews)
		if err == nil {
			clog.Views = sql.NullInt64{Int64: int64(views), Valid: true}
		} else {
			clog.Views = sql.NullInt64{Int64: int64(0), Valid: true}
			// log.Println(err, streamer.UserId)
		}

		// st, err := strconv.Atoi(ldata.LiveStart)
		st, err := time.Parse("Mon, 02 Jan 2006 15:04:05 -0700", ldata.LiveStart)
		if err == nil {
			startTime := st
			clog.LiveStartTime = sql.NullTime{Time: startTime, Valid: true}
			dt, err := strconv.Atoi(ldata.LiveDuration)
			if time.Now().Sub(startTime) >= time.Hour*24*90 {
				streamer.Operator = 5
			}

			if err == nil {
				endTime := startTime.Add((time.Duration)(dt) * time.Millisecond)
				clog.LiveEndTime = sql.NullTime{Time: endTime, Valid: true}
			} else {
				log.Println(err, streamer.UserId)
			}
		} else {
			log.Println(err, streamer.UserId)
		}

		streamer.LatestLogUid = estore.InsertClog(clog)
		estore.UpdateStreamer(streamer)
	}
}