package main import ( "database/sql" "encoding/json" "errors" "intimate" "log" "time" "github.com/474420502/extractor" "github.com/tidwall/gjson" ) var estore = intimate.NewStoreExtractor() var sstore = intimate.NewStoreSource(string(intimate.STOpenrec)) //UserInfo 提取信息的结构体 type UserInfo struct { UserName string `exp:"//p[ contains(@class, 'c-global__user__profile__list__name__text')]"` Followers int64 `exp:"//p[@class='c-global__user__count__row__right js-userCountFollowers']" mth:"r:ParseNumber"` Views int64 `exp:"//ul[@class='c-contents']//p[@class='c-thumbnailVideo__footer__liveCount']" mth:"r:ExtractNumber"` } //UserLive 提取信息的结构体 type UserLive struct { Title string `exp:"//h1[contains(@class,'MovieTitle__Title')]"` LiveStartTime string `exp:"//meta[@itemprop='uploadDate']/@content"` LiveEndTime string `exp:"//meta[@itemprop='duration']/@content"` Tags []string `exp:"//a[contains(@class,'TagButton')]"` } // Execute 执行 func Execute() { ps := intimate.NewPerfectShutdown() var lasterr error = nil for !ps.IsClose() { var err error source, err := sstore.Pop(intimate.TOpenrecUser, 0) if err != nil { if err != lasterr { log.Println(err, lasterr) lasterr = err } time.Sleep(time.Second * 5) continue } lasterr = nil sdata := source.Ext.([]byte) datamap := gjson.ParseBytes(sdata).Map() source.Operator = int32(intimate.OperatorError) userId := datamap["var_user_id"].String() streamer := &intimate.Streamer{} streamer.UserId = userId // streamer.Platform = intimate.Popenrec 不需要更新字段 htmlUser := datamap["html_user"] userEtor := extractor.ExtractHtmlString(htmlUser.String()) ui, ok1 := userEtor.GetObjectByTag(UserInfo{}).(*UserInfo) htmlLive := datamap["html_live"] liveEtor := extractor.ExtractHtmlString(htmlLive.String()) ul, ok2 := liveEtor.GetObjectByTag(UserLive{}).(*UserLive) jsonSupporters := datamap["json_supporters"] clog := &intimate.CollectLog{} if ok1 { clog.Followers = sql.NullInt64{Int64: ui.Followers, Valid: true} clog.Views = sql.NullInt64{Int64: ui.Views, Valid: true} if ui.Views != 0 { clog.IsLiveStreaming = true } streamer.UserName = sql.NullString{String: ui.UserName, Valid: true} giverjson := jsonSupporters var givers []interface{} var gratuity int64 = 0 for _, v := range giverjson.Array() { giverSource := gjson.Parse(v.String()) for _, item := range giverSource.Get("data.items").Array() { givers = append(givers, item.Map()) gratuity += item.Get("total_yells").Int() } } giversbytes, err := json.Marshal(givers) if err != nil { log.Println(err) clog.ErrorMsg = sql.NullString{String: err.Error(), Valid: true} } else { clog.Giver = giversbytes } clog.Gratuity = sql.NullInt64{Int64: gratuity, Valid: true} } else { log.Println("UserInfo may be not exists") estore.UpdateError(streamer, errors.New("UserInfo may be not exists")) continue } //log.Println(ul) if ok2 { clog.LiveTitle = sql.NullString{String: ul.Title, Valid: true} startTime, err := time.ParseInLocation("2006-01-02T15:04:05Z07:00", ul.LiveStartTime, time.Local) if err != nil { log.Println(err) } else { clog.LiveStartTime = sql.NullTime{Time: startTime.Local(), Valid: true} duration, err := intimate.ParseDuration(ul.LiveEndTime) if err != nil { log.Println(err) } else { endTime := startTime.Add(duration) clog.LiveStartTime = sql.NullTime{Time: endTime.Local(), Valid: true} } } if tags, err := json.Marshal(ul.Tags); err == nil { clog.Tags = tags } else { log.Println("json error", ul.Tags, clog.Tags) } } streamer.Uid = source.StreamerId.Int64 streamer.UpdateTime = source.UpdateTime streamer.Tags = clog.Tags clog.Platform = intimate.Popenrec clog.UserId = userId clog.UpdateTime = source.UpdateTime clog.StreamerUid = streamer.Uid logUid := estore.InsertClog(clog) LiveUrl := "https://www.openrec.tv/live/" + userId streamer.LiveUrl = sql.NullString{String: LiveUrl, Valid: true} streamer.LatestLogUid = logUid // streamer.Operator = 0 log.Println(streamer.UserId) estore.Update(streamer, "user_name", streamer.UserName, "user_id", streamer.UserId, "live_url", streamer.LiveUrl, "latest_log_uid", streamer.LatestLogUid, "update_time", streamer.UpdateTime, "tags", streamer.Tags, ) source.Operator = int32(intimate.OperatorExtractorOK) sstore.UpdateOperator(source) } }