Merge branch 'release/v0.2.0'
This commit is contained in:
eson
commit
8ba6169932
21
config.go
21
config.go
|
|
@ -3,6 +3,7 @@ package intimate
|
|||
import (
|
||||
"errors"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"gopkg.in/yaml.v2"
|
||||
|
|
@ -15,25 +16,33 @@ func init() {
|
|||
InitConfig = &Config{}
|
||||
InitConfig.Load()
|
||||
// storeOpenrec = NewStore()
|
||||
|
||||
log.SetFlags(log.Llongfile | log.Ldate)
|
||||
}
|
||||
|
||||
// Config 配置
|
||||
type Config struct {
|
||||
Database struct {
|
||||
URI string `yaml:"uri"` // "user:password@/dbname"
|
||||
SourceURI string `yaml:"source_uri"` // "user:password@/dbname"
|
||||
ExtractorURI string `yaml:"extractor_uri"`
|
||||
} `yaml:"database"`
|
||||
}
|
||||
|
||||
// Load 加载yaml/yml配置
|
||||
func (conifg *Config) Load() {
|
||||
configfile := "./config.yaml"
|
||||
if _, err := os.Stat(configfile); os.IsNotExist(err) {
|
||||
configfile = "./config.yml"
|
||||
if _, err := os.Stat(configfile); os.IsNotExist(err) {
|
||||
panic(errors.New("config.yaml or config.yml is not exists"))
|
||||
var configfile string
|
||||
configlist := []string{"./config.yaml", "./config.yml", "../../config.yml", "../../config.yaml", "../../../config.yml", "../../../config.yaml"}
|
||||
for _, configfile = range configlist {
|
||||
if _, err := os.Stat(configfile); err == nil {
|
||||
log.Println("find config: ", configfile)
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if len(configfile) <= 4 {
|
||||
log.Panic(errors.New("can't find config.yaml/config.yml"))
|
||||
}
|
||||
|
||||
f, err := os.Open(configfile)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
|
|
|
|||
|
|
@ -1,2 +1,3 @@
|
|||
database:
|
||||
uri: "root:@tcp(127.0.0.1:4000)/intimate_source"
|
||||
source_uri: "root:@tcp(127.0.0.1:4000)/intimate_source?parseTime=true"
|
||||
extractor_uri: "root:@tcp(127.0.0.1:4000)/intimate_extractor?parseTime=true"
|
||||
|
|
@ -6,7 +6,7 @@ func TestConfig(t *testing.T) {
|
|||
config := &Config{}
|
||||
config.Load()
|
||||
|
||||
if config.Database.URI != "root:@tcp(127.0.0.1:4000)/intimate_source" {
|
||||
if config.Database.SourceURI != "root:@tcp(127.0.0.1:4000)/intimate_source" {
|
||||
t.Error("error yaml loaded, ", config)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
4
extractor/openrec_extractor/.gitignore
vendored
Normal file
4
extractor/openrec_extractor/.gitignore
vendored
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
*.html
|
||||
log
|
||||
screenlog.*
|
||||
openrec_extractor
|
||||
16
extractor/openrec_extractor/main.go
Normal file
16
extractor/openrec_extractor/main.go
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
package main
|
||||
|
||||
/*
|
||||
`uid` varchar(36) NOT NULL,
|
||||
`platform` varchar(255) NOT NULL,
|
||||
`anchor_id` varchar(255) NOT NULL,
|
||||
`anchor_name` varchar(255) NOT NULL,
|
||||
`live_url` text,
|
||||
`channel` varchar(128) DEFAULT NULL, // 没有分类
|
||||
`show_type` varchar(255) DEFAULT NULL,
|
||||
*/
|
||||
|
||||
func main() {
|
||||
oe := &OpenrecExtractor{}
|
||||
oe.Execute()
|
||||
}
|
||||
244
extractor/openrec_extractor/openrec_extractor.go
Normal file
244
extractor/openrec_extractor/openrec_extractor.go
Normal file
|
|
@ -0,0 +1,244 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"intimate"
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/tidwall/gjson"
|
||||
)
|
||||
|
||||
// OpenrecExtractor 提取方法
|
||||
type OpenrecExtractor struct {
|
||||
user *intimate.ExtractorSource
|
||||
userLive *intimate.ExtractorSource
|
||||
supporters *intimate.ExtractorSource
|
||||
}
|
||||
|
||||
func (oe *OpenrecExtractor) Execute() {
|
||||
|
||||
var loop int32 = 1
|
||||
|
||||
go func() {
|
||||
signalchan := make(chan os.Signal)
|
||||
signal.Notify(signalchan, syscall.SIGKILL, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGSTOP)
|
||||
log.Println("accept stop command:", <-signalchan)
|
||||
atomic.StoreInt32(&loop, 0)
|
||||
}()
|
||||
|
||||
collect := intimate.NewExtractorStore()
|
||||
store := intimate.NewSourceStore("source_openrec")
|
||||
var lasterr error = nil
|
||||
|
||||
for atomic.LoadInt32(&loop) > 0 {
|
||||
|
||||
source, err := store.Pop(string(intimate.TTOpenrecRanking), 100)
|
||||
if err != nil {
|
||||
if err != lasterr {
|
||||
log.Println(err, lasterr)
|
||||
lasterr = err
|
||||
}
|
||||
time.Sleep(time.Second * 2)
|
||||
continue
|
||||
}
|
||||
|
||||
source.SetOperator(int32(intimate.OperatorError))
|
||||
anchorId := source.GetSource().String
|
||||
|
||||
ai := &intimate.AnchorInfo{}
|
||||
ai.SetAnchorId(anchorId)
|
||||
ai.SetPlatform(string(intimate.Popenrec))
|
||||
|
||||
sdata := source.GetExt().([]byte)
|
||||
if gjson.ValidBytes(sdata) {
|
||||
result := gjson.ParseBytes(sdata)
|
||||
datamap := result.Map()
|
||||
|
||||
oe.user = intimate.NewExtractorSource(datamap["user"])
|
||||
oe.user.CreateExtractor()
|
||||
|
||||
oe.userLive = intimate.NewExtractorSource(datamap["user_live"])
|
||||
oe.userLive.CreateExtractor()
|
||||
|
||||
oe.supporters = intimate.NewExtractorSource(datamap["supporters"])
|
||||
|
||||
clog := &intimate.CollectLog{}
|
||||
|
||||
log.Println(anchorId)
|
||||
|
||||
oe.extractFollowers(clog)
|
||||
oe.extractAnchorName(ai)
|
||||
oe.extractViewsAndLiveStreaming(clog)
|
||||
oe.extractGiversAndGratuity(clog)
|
||||
oe.extractLive(clog)
|
||||
oe.extractTags(clog)
|
||||
|
||||
ai.Set("UpdateTime", source.GetUpdateTime())
|
||||
|
||||
LiveUrl := "https://www.openrec.tv/live/" + anchorId
|
||||
ai.Set("LiveUrl", sql.NullString{String: LiveUrl, Valid: true})
|
||||
|
||||
Uid, err := collect.InsertAnchorInfo(ai)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
source.SetErrorMsg(sql.NullString{String: err.Error(), Valid: true})
|
||||
store.UpdateOperator(source)
|
||||
return
|
||||
}
|
||||
|
||||
clog.Set("Uid", Uid)
|
||||
clog.Set("Platform", string(intimate.Popenrec))
|
||||
clog.Set("AnchorId", anchorId)
|
||||
clog.Set("UpdateTime", source.GetUpdateTime())
|
||||
|
||||
if err = collect.InsertCollectLog(clog); err != nil {
|
||||
source.SetErrorMsg(sql.NullString{String: err.Error(), Valid: true})
|
||||
store.UpdateOperator(source)
|
||||
return
|
||||
}
|
||||
|
||||
source.SetOperator(int32(intimate.OperatorExtractorOK))
|
||||
store.UpdateOperator(source)
|
||||
} else {
|
||||
log.Println("data is not json:\n", string(sdata))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (oe *OpenrecExtractor) extractFollowers(clog intimate.ISet) {
|
||||
extractor := oe.user.GetExtractor()
|
||||
xp, err := extractor.XPathResult("//p[@class='c-global__user__count__row__right js-userCountFollowers']/text()")
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
if !xp.NodeIter().Next() {
|
||||
log.Println("不存在粉丝数")
|
||||
}
|
||||
|
||||
followers := strings.ReplaceAll(xp.String(), ",", "")
|
||||
followersInt, err := strconv.ParseInt(followers, 10, 64)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
|
||||
clog.Set("Followers", sql.NullInt64{Int64: followersInt, Valid: true})
|
||||
}
|
||||
|
||||
func (oe *OpenrecExtractor) extractAnchorName(ai intimate.ISet) {
|
||||
extractor := oe.user.GetExtractor()
|
||||
xp, err := extractor.XPathResult("//p[@class='c-global__user__profile__list__name__text official-icon--after']/text()")
|
||||
if xp.NodeIter().Next() {
|
||||
anchorName := xp.String()
|
||||
ai.Set("AnchorName", anchorName)
|
||||
} else {
|
||||
log.Println(err)
|
||||
}
|
||||
}
|
||||
|
||||
func (oe *OpenrecExtractor) extractViewsAndLiveStreaming(clog intimate.ISet) {
|
||||
extractor := oe.user.GetExtractor()
|
||||
// c-contents
|
||||
xp, err := extractor.XPathResult("//ul[@class='c-contents']//p[@class='c-thumbnailVideo__footer__liveCount']/text()")
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
if xp.NodeIter().Next() {
|
||||
views := regexp.MustCompile(`[0-9,]+`).FindString(xp.String())
|
||||
views = strings.ReplaceAll(views, ",", "")
|
||||
viewsint, err := strconv.Atoi(views)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
|
||||
clog.Set("Views", sql.NullInt64{Int64: int64(viewsint), Valid: true})
|
||||
clog.Set("IsLiveStreaming", int32(1))
|
||||
}
|
||||
}
|
||||
|
||||
func (oe *OpenrecExtractor) extractGiversAndGratuity(clog intimate.ISet) {
|
||||
// extractor := oe.user.GetExtractor()
|
||||
giverjson := oe.supporters.GetSource()
|
||||
var givers []interface{}
|
||||
var gratuity int64 = 0
|
||||
|
||||
for _, v := range giverjson.Array() {
|
||||
giverSource := gjson.Parse(v.String())
|
||||
for _, item := range giverSource.Get("data.items").Array() {
|
||||
givers = append(givers, item.Map())
|
||||
gratuity += item.Get("total_yells").Int()
|
||||
}
|
||||
}
|
||||
|
||||
giversbytes, err := json.Marshal(givers)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
clog.Set("ErrorMsg", sql.NullString{String: err.Error(), Valid: true})
|
||||
} else {
|
||||
clog.Set("Giver", giversbytes)
|
||||
}
|
||||
|
||||
clog.Set("Gratuity", sql.NullInt64{Int64: gratuity, Valid: true})
|
||||
}
|
||||
|
||||
func (oe *OpenrecExtractor) extractLive(clog intimate.ISet) {
|
||||
extractor := oe.userLive.GetExtractor()
|
||||
mathes := regexp.MustCompile("MovieTitle__Title[^>]+>(.{1,50})</h1>").FindStringSubmatch(oe.userLive.GetSource().Str)
|
||||
if len(mathes) == 2 {
|
||||
|
||||
clog.Set("LiveTitle", sql.NullString{String: mathes[1], Valid: true})
|
||||
|
||||
content, err := extractor.XPathResult("//meta[@itemprop='uploadDate']/@content")
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
|
||||
iter := content.NodeIter()
|
||||
if iter.Next() {
|
||||
tm, err := time.ParseInLocation("2006-01-02T15:04:05Z07:00", iter.Node().NodeValue(), time.Local)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
clog.Set("LiveStartTime", sql.NullTime{Time: tm.Local(), Valid: true})
|
||||
|
||||
duration, err := extractor.XPathResult("//meta[@itemprop='duration']/@content")
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
|
||||
diter := duration.NodeIter()
|
||||
if diter.Next() {
|
||||
|
||||
dt, err := intimate.ParseDuration(diter.Node().NodeValue())
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
endtm := tm.Add(dt)
|
||||
clog.Set("LiveEndTime", sql.NullTime{Time: endtm.Local(), Valid: true})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (oe *OpenrecExtractor) extractTags(clog intimate.ISet) {
|
||||
var tags []string
|
||||
matheslist := regexp.MustCompile(`<[^>]+TagButton[^>]+>([^<]{1,100})<`).FindAllStringSubmatch(oe.userLive.GetSource().Str, -1)
|
||||
for _, m := range matheslist {
|
||||
tags = append(tags, m[1])
|
||||
}
|
||||
log.Println(tags)
|
||||
tagsBytes, err := json.Marshal(tags)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
|
||||
clog.Set("Tags", tagsBytes)
|
||||
}
|
||||
96
extractor/openrec_extractor/openrec_test.go
Normal file
96
extractor/openrec_extractor/openrec_test.go
Normal file
|
|
@ -0,0 +1,96 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"regexp"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/lestrrat-go/libxml2"
|
||||
)
|
||||
|
||||
func TestCase0(t *testing.T) {
|
||||
f, err := os.Open("./test.html")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
data, err := ioutil.ReadAll(f)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
matheslist := regexp.MustCompile(`TagButton__Button[^>]+>(.{1,100})</a`).FindAllStringSubmatch(string(data), -1)
|
||||
t.Error(matheslist)
|
||||
}
|
||||
|
||||
func TestCase1(t *testing.T) {
|
||||
date := "2020-07-13T18:58:24+09:00"
|
||||
|
||||
tm, err := time.Parse("2006-01-02T15:04:05Z07:00", date)
|
||||
t.Error(err)
|
||||
t.Error(time.Now())
|
||||
t.Error(tm.Local().UTC(), tm.Local())
|
||||
|
||||
}
|
||||
|
||||
func TestCase2(t *testing.T) {
|
||||
duration1 := "0:00:00"
|
||||
duration2 := "4:56:04"
|
||||
tm2, err := time.Parse("15:04:05", duration2)
|
||||
tm1, err := time.Parse("15:04:05", duration1)
|
||||
|
||||
tm2.Sub(tm1)
|
||||
|
||||
t.Error(err)
|
||||
t.Error(tm2.Sub(tm1))
|
||||
|
||||
}
|
||||
|
||||
func TestCase(t *testing.T) {
|
||||
f, _ := os.Open("./test.html")
|
||||
data, _ := ioutil.ReadAll(f)
|
||||
|
||||
doc, err := libxml2.ParseHTML(data)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
// doc.CreateElement("meta")
|
||||
// "<META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=utf-8">"
|
||||
|
||||
xresult, err := doc.Find("/html/head")
|
||||
ele, err := doc.CreateElement(`META`)
|
||||
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
ele.SetAttribute("charset", "utf-8")
|
||||
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
iter := xresult.NodeIter()
|
||||
if iter.Next() {
|
||||
n := iter.Node()
|
||||
|
||||
err = n.AddChild(ele)
|
||||
// childs, err := n.ChildNodes()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
t.Error(n)
|
||||
}
|
||||
|
||||
xr, err := doc.Find("//h1[ contains(@class, 'MovieTitle__Title')]")
|
||||
if err != nil {
|
||||
panic(nil)
|
||||
}
|
||||
|
||||
t.Error(xr)
|
||||
}
|
||||
|
||||
func TestExtractor(t *testing.T) {
|
||||
oe := &OpenrecExtractor{}
|
||||
oe.Execute()
|
||||
}
|
||||
385
extractor_field.go
Normal file
385
extractor_field.go
Normal file
|
|
@ -0,0 +1,385 @@
|
|||
package intimate
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"reflect"
|
||||
|
||||
"github.com/474420502/hunter"
|
||||
"github.com/tidwall/gjson"
|
||||
)
|
||||
|
||||
type ISetAnchorInfo interface {
|
||||
SetUid(int64) //
|
||||
SetPlatform(string) //
|
||||
SetAnchorId(string) //
|
||||
SetAnchorName(string) //
|
||||
SetLiveUrl(sql.NullString) //
|
||||
SetChannel(sql.NullString) //
|
||||
SetTags(interface{}) //
|
||||
SetExt(interface{}) //
|
||||
SetUpdateTime(sql.NullTime) //
|
||||
}
|
||||
|
||||
type IGetAnchorInfo interface {
|
||||
GetUid() int64 //
|
||||
GetPlatform() string //
|
||||
GetAnchorId() string //
|
||||
GetAnchorName() string //
|
||||
GetLiveUrl() sql.NullString //
|
||||
GetChannel() sql.NullString //
|
||||
GetTags() interface{}
|
||||
GetExt() interface{} //
|
||||
GetUpdateTime() sql.NullTime //
|
||||
}
|
||||
|
||||
type AnchorInfo struct {
|
||||
Uid int64 //
|
||||
Platform string //
|
||||
AnchorId string //
|
||||
AnchorName string //
|
||||
LiveUrl sql.NullString //
|
||||
Channel sql.NullString //
|
||||
Tags interface{}
|
||||
Ext interface{} //
|
||||
UpdateTime sql.NullTime //
|
||||
}
|
||||
|
||||
// Set Simple Value
|
||||
func (ai *AnchorInfo) Set(field string, value interface{}) {
|
||||
reflect.ValueOf(ai).Elem().FieldByName(field).Set(reflect.ValueOf(value))
|
||||
}
|
||||
|
||||
// GetTags Get return Tags interface{}
|
||||
func (ai *AnchorInfo) GetTags() interface{} {
|
||||
return ai.Tags
|
||||
}
|
||||
|
||||
// SetTags Set Tags interface{}
|
||||
func (ai *AnchorInfo) SetTags(Tags interface{}) {
|
||||
ai.Tags = Tags
|
||||
}
|
||||
|
||||
// GetUpdateTime Get return UpdateTime time.Time
|
||||
func (ai *AnchorInfo) GetUpdateTime() sql.NullTime {
|
||||
return ai.UpdateTime
|
||||
}
|
||||
|
||||
// SetUpdateTime Set UpdateTime time.Time
|
||||
func (ai *AnchorInfo) SetUpdateTime(UpdateTime sql.NullTime) {
|
||||
ai.UpdateTime = UpdateTime
|
||||
}
|
||||
|
||||
// GetExt Get return Ext interface{}
|
||||
func (ai *AnchorInfo) GetExt() interface{} {
|
||||
return ai.Ext
|
||||
}
|
||||
|
||||
// SetExt Set Ext interface{}
|
||||
func (ai *AnchorInfo) SetExt(Ext interface{}) {
|
||||
ai.Ext = Ext
|
||||
}
|
||||
|
||||
// GetChannel Get return Channel sql.NullString
|
||||
func (ai *AnchorInfo) GetChannel() sql.NullString {
|
||||
return ai.Channel
|
||||
}
|
||||
|
||||
// SetChannel Set Channel sql.NullString
|
||||
func (ai *AnchorInfo) SetChannel(Channel sql.NullString) {
|
||||
ai.Channel = Channel
|
||||
}
|
||||
|
||||
// GetLiveUrl Get return LiveUrl sql.NullString
|
||||
func (ai *AnchorInfo) GetLiveUrl() sql.NullString {
|
||||
return ai.LiveUrl
|
||||
}
|
||||
|
||||
// SetLiveUrl Set LiveUrl sql.NullString
|
||||
func (ai *AnchorInfo) SetLiveUrl(LiveUrl sql.NullString) {
|
||||
ai.LiveUrl = LiveUrl
|
||||
}
|
||||
|
||||
// GetAnchorName Get return AnchorName string
|
||||
func (ai *AnchorInfo) GetAnchorName() string {
|
||||
return ai.AnchorName
|
||||
}
|
||||
|
||||
// SetAnchorName Set AnchorName string
|
||||
func (ai *AnchorInfo) SetAnchorName(AnchorName string) {
|
||||
ai.AnchorName = AnchorName
|
||||
}
|
||||
|
||||
// GetAnchorId Get return AnchorId string
|
||||
func (ai *AnchorInfo) GetAnchorId() string {
|
||||
return ai.AnchorId
|
||||
}
|
||||
|
||||
// SetAnchorId Set AnchorId string
|
||||
func (ai *AnchorInfo) SetAnchorId(AnchorId string) {
|
||||
ai.AnchorId = AnchorId
|
||||
}
|
||||
|
||||
// GetPlatform Get return Platform string
|
||||
func (ai *AnchorInfo) GetPlatform() string {
|
||||
return ai.Platform
|
||||
}
|
||||
|
||||
// SetPlatform Set Platform string
|
||||
func (ai *AnchorInfo) SetPlatform(Platform string) {
|
||||
ai.Platform = Platform
|
||||
}
|
||||
|
||||
// GetUid Get return Uid int64
|
||||
func (ai *AnchorInfo) GetUid() int64 {
|
||||
return ai.Uid
|
||||
}
|
||||
|
||||
// SetUid Set Uid int64
|
||||
func (ai *AnchorInfo) SetUid(Uid int64) {
|
||||
ai.Uid = Uid
|
||||
}
|
||||
|
||||
type IGetCollectLog interface {
|
||||
GetUid() int64 //
|
||||
GetPlatform() string //
|
||||
GetAnchorId() string //
|
||||
GetIsLiveStreaming() int32 //
|
||||
GetIsError() int32 //
|
||||
GetFollowers() sql.NullInt64 //
|
||||
GetViews() sql.NullInt64 //
|
||||
GetGiver() interface{} //
|
||||
GetGratuity() sql.NullInt64 //
|
||||
GetLiveTitle() sql.NullString //
|
||||
GetLiveStartTime() sql.NullTime //
|
||||
GetLiveEndTime() sql.NullTime //
|
||||
GetUpdateTime() sql.NullTime //
|
||||
GetTags() interface{} //
|
||||
GetExt() interface{} //
|
||||
GetErrorMsg() sql.NullString //
|
||||
}
|
||||
|
||||
type ISetCollectLog interface {
|
||||
SetUid(int64) //
|
||||
SetPlatform(string) //
|
||||
SetAnchorId(string) //
|
||||
SetIsLiveStreaming(int32) //
|
||||
SetIsError(int32) //
|
||||
SetFollowers(sql.NullInt64) //
|
||||
SetViews(sql.NullInt64) //
|
||||
SetGiver(interface{}) //
|
||||
SetGratuity(sql.NullInt64) //
|
||||
SetLiveTitle(sql.NullString) //
|
||||
SetLiveStartTime(sql.NullTime) //
|
||||
SetLiveEndTime(sql.NullTime) //
|
||||
SetUpdateTime(sql.NullTime) //
|
||||
SetTags(interface{}) //
|
||||
SetExt(interface{}) //
|
||||
SetErrorMsg(sql.NullString) //
|
||||
}
|
||||
|
||||
type CollectLog struct {
|
||||
Uid int64 //
|
||||
Platform string //
|
||||
AnchorId string //
|
||||
IsLiveStreaming int32 //
|
||||
IsError int32 //
|
||||
Followers sql.NullInt64 //
|
||||
Views sql.NullInt64 //
|
||||
Giver interface{} //
|
||||
Gratuity sql.NullInt64 //
|
||||
LiveTitle sql.NullString //
|
||||
LiveStartTime sql.NullTime //
|
||||
LiveEndTime sql.NullTime //
|
||||
UpdateTime sql.NullTime //
|
||||
Tags interface{}
|
||||
Ext interface{} //
|
||||
ErrorMsg sql.NullString //
|
||||
}
|
||||
|
||||
// Set Simple Value
|
||||
func (cl *CollectLog) Set(field string, value interface{}) {
|
||||
reflect.ValueOf(cl).Elem().FieldByName(field).Set(reflect.ValueOf(value))
|
||||
}
|
||||
|
||||
// GetTags Get return Tags interface{}
|
||||
func (cl *CollectLog) GetTags() interface{} {
|
||||
return cl.Tags
|
||||
}
|
||||
|
||||
// SetTags Set Tags interface{}
|
||||
func (cl *CollectLog) SetTags(Tags interface{}) {
|
||||
cl.Tags = Tags
|
||||
}
|
||||
|
||||
// GetErrorMsg Get return Error sql.NullString
|
||||
func (cl *CollectLog) GetErrorMsg() sql.NullString {
|
||||
return cl.ErrorMsg
|
||||
}
|
||||
|
||||
// SetErrorMsg Set Error sql.NullString
|
||||
func (cl *CollectLog) SetErrorMsg(ErrorMsg sql.NullString) {
|
||||
cl.ErrorMsg = ErrorMsg
|
||||
}
|
||||
|
||||
// GetExt Get return Ext interface{}
|
||||
func (cl *CollectLog) GetExt() interface{} {
|
||||
return cl.Ext
|
||||
}
|
||||
|
||||
// SetExt Set Ext interface{}
|
||||
func (cl *CollectLog) SetExt(Ext interface{}) {
|
||||
cl.Ext = Ext
|
||||
}
|
||||
|
||||
// GetUpdateTime Get return UpdateTime time.Time
|
||||
func (cl *CollectLog) GetUpdateTime() sql.NullTime {
|
||||
return cl.UpdateTime
|
||||
}
|
||||
|
||||
// SetUpdateTime Set UpdateTime time.Time
|
||||
func (cl *CollectLog) SetUpdateTime(UpdateTime sql.NullTime) {
|
||||
cl.UpdateTime = UpdateTime
|
||||
}
|
||||
|
||||
// GetLiveEndTime Get return ShowEndTime sql.NullTime
|
||||
func (cl *CollectLog) GetLiveEndTime() sql.NullTime {
|
||||
return cl.LiveEndTime
|
||||
}
|
||||
|
||||
// SetLiveEndTime Set ShowEndTime sql.NullTime
|
||||
func (cl *CollectLog) SetLiveEndTime(ShowEndTime sql.NullTime) {
|
||||
cl.LiveEndTime = ShowEndTime
|
||||
}
|
||||
|
||||
// GetLiveStartTime Get return ShowStartTime sql.NullTime
|
||||
func (cl *CollectLog) GetLiveStartTime() sql.NullTime {
|
||||
return cl.LiveStartTime
|
||||
}
|
||||
|
||||
// SetLiveStartTime Set ShowStartTime sql.NullTime
|
||||
func (cl *CollectLog) SetLiveStartTime(ShowStartTime sql.NullTime) {
|
||||
cl.LiveStartTime = ShowStartTime
|
||||
}
|
||||
|
||||
// GetLiveTitle Get return ShowTitle sql.NullString
|
||||
func (cl *CollectLog) GetLiveTitle() sql.NullString {
|
||||
return cl.LiveTitle
|
||||
}
|
||||
|
||||
// SetLiveTitle Set ShowTitle sql.NullString
|
||||
func (cl *CollectLog) SetLiveTitle(ShowTitle sql.NullString) {
|
||||
cl.LiveTitle = ShowTitle
|
||||
}
|
||||
|
||||
// GetGratuity Get return Gratuity sql.NullInt32
|
||||
func (cl *CollectLog) GetGratuity() sql.NullInt64 {
|
||||
return cl.Gratuity
|
||||
}
|
||||
|
||||
// SetGratuity Set Gratuity sql.NullInt32
|
||||
func (cl *CollectLog) SetGratuity(Gratuity sql.NullInt64) {
|
||||
cl.Gratuity = Gratuity
|
||||
}
|
||||
|
||||
// GetGiver Get return Giver interface{}
|
||||
func (cl *CollectLog) GetGiver() interface{} {
|
||||
return cl.Giver
|
||||
}
|
||||
|
||||
// SetGiver Set Giver interface{}
|
||||
func (cl *CollectLog) SetGiver(Giver interface{}) {
|
||||
cl.Giver = Giver
|
||||
}
|
||||
|
||||
// GetViews Get return Views sql.NullInt64
|
||||
func (cl *CollectLog) GetViews() sql.NullInt64 {
|
||||
return cl.Views
|
||||
}
|
||||
|
||||
// SetViews Set Views sql.NullInt64
|
||||
func (cl *CollectLog) SetViews(Views sql.NullInt64) {
|
||||
cl.Views = Views
|
||||
}
|
||||
|
||||
// GetFollowers Get return Followers sql.NullInt64
|
||||
func (cl *CollectLog) GetFollowers() sql.NullInt64 {
|
||||
return cl.Followers
|
||||
}
|
||||
|
||||
// SetFollowers Set Followers sql.NullInt32
|
||||
func (cl *CollectLog) SetFollowers(Followers sql.NullInt64) {
|
||||
cl.Followers = Followers
|
||||
}
|
||||
|
||||
// GetIsError Get return IsError int32
|
||||
func (cl *CollectLog) GetIsError() int32 {
|
||||
return cl.IsError
|
||||
}
|
||||
|
||||
// SetIsError Set IsError int32
|
||||
func (cl *CollectLog) SetIsError(IsError int32) {
|
||||
cl.IsError = IsError
|
||||
}
|
||||
|
||||
// GetIsLiveStreaming Get return IsShowing int32
|
||||
func (cl *CollectLog) GetIsLiveStreaming() int32 {
|
||||
return cl.IsLiveStreaming
|
||||
}
|
||||
|
||||
// SetIsLiveStreaming Set IsShowing int32
|
||||
func (cl *CollectLog) SetIsLiveStreaming(IsLive int32) {
|
||||
cl.IsLiveStreaming = IsLive
|
||||
}
|
||||
|
||||
// GetAnchorId Get return AnchorId string
|
||||
func (cl *CollectLog) GetAnchorId() string {
|
||||
return cl.AnchorId
|
||||
}
|
||||
|
||||
// SetAnchorId Set AnchorId string
|
||||
func (cl *CollectLog) SetAnchorId(AnchorId string) {
|
||||
cl.AnchorId = AnchorId
|
||||
}
|
||||
|
||||
// GetPlatform Get return Platform string
|
||||
func (cl *CollectLog) GetPlatform() string {
|
||||
return cl.Platform
|
||||
}
|
||||
|
||||
// SetPlatform Set Platform string
|
||||
func (cl *CollectLog) SetPlatform(Platform string) {
|
||||
cl.Platform = Platform
|
||||
}
|
||||
|
||||
// GetUid Get return Uid int64
|
||||
func (cl *CollectLog) GetUid() int64 {
|
||||
return cl.Uid
|
||||
}
|
||||
|
||||
// SetUid Set Uid int64
|
||||
func (cl *CollectLog) SetUid(Uid int64) {
|
||||
cl.Uid = Uid
|
||||
}
|
||||
|
||||
type ExtractorSource struct {
|
||||
source gjson.Result
|
||||
extractor *hunter.Extractor
|
||||
}
|
||||
|
||||
func NewExtractorSource(gr gjson.Result) *ExtractorSource {
|
||||
es := &ExtractorSource{}
|
||||
es.source = gr
|
||||
return es
|
||||
}
|
||||
|
||||
func (es *ExtractorSource) CreateExtractor() {
|
||||
es.extractor = hunter.NewExtractor([]byte(es.source.Str))
|
||||
}
|
||||
|
||||
func (es *ExtractorSource) GetSource() gjson.Result {
|
||||
return es.source
|
||||
}
|
||||
|
||||
func (es *ExtractorSource) GetExtractor() *hunter.Extractor {
|
||||
return es.extractor
|
||||
}
|
||||
11
go.mod
11
go.mod
|
|
@ -3,13 +3,12 @@ module intimate
|
|||
go 1.14
|
||||
|
||||
require (
|
||||
github.com/474420502/hunter v0.1.2
|
||||
github.com/474420502/gcurl v0.1.2
|
||||
github.com/474420502/hunter v0.3.0
|
||||
github.com/go-sql-driver/mysql v1.5.0
|
||||
github.com/go-yaml/yaml v2.1.0+incompatible // indirect
|
||||
github.com/satori/go.uuid v1.2.0
|
||||
github.com/lestrrat-go/libxml2 v0.0.0-20200215080510-6483566f52cb
|
||||
github.com/tidwall/gjson v1.6.0
|
||||
github.com/tidwall/pretty v1.0.1 // indirect
|
||||
golang.org/x/tools v0.0.0-20200708003708-134513de8882 // indirect
|
||||
gopkg.in/yaml.v2 v2.2.2
|
||||
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a
|
||||
golang.org/x/net v0.0.0-20200707034311-ab3426394381 // indirect
|
||||
gopkg.in/yaml.v2 v2.3.0
|
||||
)
|
||||
|
|
|
|||
55
go.sum
55
go.sum
|
|
@ -2,21 +2,25 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT
|
|||
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
|
||||
cloud.google.com/go v0.41.0/go.mod h1:OauMR7DV8fzvZIl2qg6rkaIhD/vmgk4iwEw/h6ercmg=
|
||||
github.com/474420502/focus v0.9.0 h1:Y/TjSdSdIgegO78OJixphiPl1wVqhK9AbcXjiDDugo4=
|
||||
github.com/474420502/focus v0.9.0/go.mod h1:jrDXvK1CnUJ3PCR3ZJVYinbS2Yz5kM8OoAbCLe6AF7Y=
|
||||
github.com/474420502/gcurl v0.0.4 h1:eR1BNXvQ4T245dotWpjDzAMWch+FTTfScqzsdq93JK0=
|
||||
github.com/474420502/gcurl v0.0.4/go.mod h1:qtCzAZZbVRIsBt0lNUh2I0qDniU9T3E21aSsVUYo7Hc=
|
||||
github.com/474420502/hunter v0.1.2 h1:1dY2C9IJvh81+04Xx8OuTPcUw6WNlKCiOaGmJdgX83Y=
|
||||
github.com/474420502/hunter v0.1.2/go.mod h1:41DpZWSsGBWsFwb/liapbT1uH58Yvl+BpW4SJwLC2Fw=
|
||||
github.com/474420502/requests v1.5.0/go.mod h1:SLXrQ5dL9c7dkIeKNUCBAjOIt3J9KFCS2RQjWJecNwo=
|
||||
github.com/474420502/requests v1.5.1 h1:miv6O4RMbZ8I0ZdUTLf/EU5Dmewc/4IL/DmUMwtuv8M=
|
||||
github.com/474420502/requests v1.5.1/go.mod h1:SLXrQ5dL9c7dkIeKNUCBAjOIt3J9KFCS2RQjWJecNwo=
|
||||
github.com/474420502/focus v0.12.0 h1:+icbmj7IEOefvTegHt5EpcHt6WFbe2miIrceUJx2Evo=
|
||||
github.com/474420502/focus v0.12.0/go.mod h1:d0PMjtMxFz1a9HIhwyFPkWa+JF+0LgOrEUfd8iZka6s=
|
||||
github.com/474420502/gcurl v0.1.2 h1:ON9Yz3IgAdtDlFlHfkAJ3aIEBDxH0RiViPE5ST5ohKg=
|
||||
github.com/474420502/gcurl v0.1.2/go.mod h1:hws5q/Ao64bXLLDnldz9VyTQUndTWc/i5DzdEazFfoM=
|
||||
github.com/474420502/hunter v0.3.0 h1:0VPi1MInxjHOta3da4v0ALWK0y3/X4/6nUSLFvdbiFU=
|
||||
github.com/474420502/hunter v0.3.0/go.mod h1:pe4Xr/I+2agvq339vS/OZV+EiHAWtpXQs75rioSW9oA=
|
||||
github.com/474420502/requests v1.6.0 h1:f4h4j40eT0P5whhg9LdkotD8CaKjtuDu/vz9iSUkCgY=
|
||||
github.com/474420502/requests v1.6.0/go.mod h1:SLXrQ5dL9c7dkIeKNUCBAjOIt3J9KFCS2RQjWJecNwo=
|
||||
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802 h1:1BDTz0u9nC3//pOCMdNH+CiXJVYJh5UQNCOBG7jbELc=
|
||||
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
|
||||
github.com/BurntSushi/xgbutil v0.0.0-20160919175755-f7c97cef3b4e h1:4ZrkT/RzpnROylmoQL57iVUL57wGKTR5O6KpVnbm2tA=
|
||||
github.com/BurntSushi/xgbutil v0.0.0-20160919175755-f7c97cef3b4e/go.mod h1:uw9h2sd4WWHOPdJ13MQpwK5qYWKYDumDqxWWIknEQ+k=
|
||||
github.com/Pallinder/go-randomdata v1.1.0 h1:gUubB1IEUliFmzjqjhf+bgkg1o6uoFIkRsP3VrhEcx8=
|
||||
github.com/Pallinder/go-randomdata v1.1.0/go.mod h1:yHmJgulpD2Nfrm0cR9tI/+oAgRqCQQixsA8HyRZfV9Y=
|
||||
github.com/Pallinder/go-randomdata v1.2.0 h1:DZ41wBchNRb/0GfsePLiSwb0PHZmT67XY00lCDlaYPg=
|
||||
github.com/Pallinder/go-randomdata v1.2.0/go.mod h1:yHmJgulpD2Nfrm0cR9tI/+oAgRqCQQixsA8HyRZfV9Y=
|
||||
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
|
||||
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
|
||||
github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
|
||||
github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
|
||||
|
|
@ -24,12 +28,13 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
|
|||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/elazarl/goproxy v0.0.0-20190711103511-473e67f1d7d2 h1:aZtFdDNWY/yH86JPR2WX/PN63635VsE/f/nXNPAbYxY=
|
||||
github.com/elazarl/goproxy v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
|
||||
github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2 h1:dWB6v3RcOy03t/bUadywsbyrQwCqZeNIEX6M1OtSZOM=
|
||||
github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8=
|
||||
github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
|
||||
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
|
||||
github.com/go-yaml/yaml v2.1.0+incompatible h1:RYi2hDdss1u4YE7GwixGzWwVo47T8UQwnTLB6vQiq+o=
|
||||
github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0=
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
||||
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
|
|
@ -39,6 +44,7 @@ github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y
|
|||
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
||||
github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
|
||||
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-github/v27 v27.0.4/go.mod h1:/0Gr8pJ55COkmv+S/yPKCczSkUPIM/LnFyubufRNIS0=
|
||||
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
|
||||
|
|
@ -54,11 +60,11 @@ github.com/lestrrat-go/libxml2 v0.0.0-20200215080510-6483566f52cb h1:qqNmX9V9n4b
|
|||
github.com/lestrrat-go/libxml2 v0.0.0-20200215080510-6483566f52cb/go.mod h1:fy/ZVbgyB83mtricxwSW3zqIRXWOVpKG2PvdUDFeC58=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc=
|
||||
github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
|
||||
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/tebeka/selenium v0.9.9 h1:cNziB+etNgyH/7KlNI7RMC1ua5aH1+5wUlFQyzeMh+w=
|
||||
github.com/tebeka/selenium v0.9.9/go.mod h1:5Fr8+pUvU6B1OiPfkdCKdXZyr5znvVkxuPd0NOdZCQc=
|
||||
|
|
@ -71,12 +77,10 @@ github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
|
|||
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
|
||||
github.com/tidwall/pretty v1.0.1 h1:WE4RBSZ1x6McVVC8S/Md+Qse8YUv6HRObAx6ke00NY8=
|
||||
github.com/tidwall/pretty v1.0.1/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
|
||||
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
|
||||
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
|
||||
|
|
@ -87,7 +91,6 @@ golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTk
|
|||
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
|
||||
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
|
|
@ -100,8 +103,8 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR
|
|||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k=
|
||||
golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20200625001655-4c5254603344 h1:vGXIOMxbNfDTk/aXCmfdLgkrSV+Z2tcbze+pEc3v5W4=
|
||||
golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU=
|
||||
golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
|
|
@ -110,7 +113,6 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ
|
|||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
|
|
@ -122,6 +124,7 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w
|
|||
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
|
|
@ -136,12 +139,6 @@ golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBn
|
|||
golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
||||
golang.org/x/tools v0.0.0-20190624190245-7f2218787638 h1:uIfBkD8gLczr4XDgYpt/qJYds2YJwZRNw4zs7wSnNhk=
|
||||
golang.org/x/tools v0.0.0-20190624190245-7f2218787638/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20200708003708-134513de8882 h1:x4Two2lSwHxTqR+eal4lB4ydUnTvmDDpPQeL92ZHDgA=
|
||||
golang.org/x/tools v0.0.0-20200708003708-134513de8882/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
|
||||
google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
|
|
@ -159,10 +156,18 @@ google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiq
|
|||
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/xmlpath.v1 v1.0.0-20140413065638-a146725ea6e7 h1:zibSPXbkfB1Dwl76rJgLa68xcdHu42qmFTe6vAnU4wA=
|
||||
gopkg.in/xmlpath.v1 v1.0.0-20140413065638-a146725ea6e7/go.mod h1:wo0SW5T6XqIKCCAge330Cd5sm+7VI6v85OrQHIk50KM=
|
||||
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
|
||||
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a h1:LJwr7TCTghdatWv40WobzlKXc9c4s8oGa7QKJUtHhWA=
|
||||
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
launchpad.net/gocheck v0.0.0-20140225173054-000000000087 h1:Izowp2XBH6Ya6rv+hqbceQyw/gSGoXfH/UPoTGduL54=
|
||||
launchpad.net/gocheck v0.0.0-20140225173054-000000000087/go.mod h1:hj7XX3B/0A+80Vse0e+BUHsHMTEhd0O4cpUHr/e/BUM=
|
||||
launchpad.net/xmlpath v0.0.0-20130614043138-000000000004 h1:B8nNZBUrx8YufDCAJjvO/lVs4GxXMQHyrjwJdJzXMFg=
|
||||
launchpad.net/xmlpath v0.0.0-20130614043138-000000000004/go.mod h1:vqyExLOM3qBx7mvYRkoxjSCF945s0mbe7YynlKYXtsA=
|
||||
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
|
||||
|
|
|
|||
9
platform_list.go
Normal file
9
platform_list.go
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
package intimate
|
||||
|
||||
// Platform 源的table列表
|
||||
type Platform string
|
||||
|
||||
const (
|
||||
// Popenrec openrec源table名称
|
||||
Popenrec Platform = "openrec"
|
||||
)
|
||||
|
|
@ -2,19 +2,66 @@ package intimate
|
|||
|
||||
import (
|
||||
"database/sql"
|
||||
"time"
|
||||
)
|
||||
|
||||
// IGetSource 源接口结构
|
||||
type IGetSource interface {
|
||||
GetUid() int64 //
|
||||
GetUrl() string //
|
||||
GetTargetType() string //
|
||||
GetSource() sql.NullString //
|
||||
GetPassGob() sql.NullString //
|
||||
GetExt() interface{} //
|
||||
GetUpdateTime() sql.NullTime //
|
||||
GetOperator() int32 //
|
||||
GetErrorMsg() sql.NullString //
|
||||
}
|
||||
|
||||
type IUpdateSource interface {
|
||||
IGetSource
|
||||
|
||||
GetLastOperator() int32
|
||||
|
||||
SetPassGob(sql.NullString)
|
||||
SetExt(ext interface{}) //
|
||||
SetUpdateTime(ut sql.NullTime) //
|
||||
SetOperator(operator int32) //
|
||||
SetErrorMsg(emsg sql.NullString) //
|
||||
}
|
||||
|
||||
// Source 的结构体
|
||||
type Source struct {
|
||||
Uid int64 //
|
||||
Url string //
|
||||
TargetType string //
|
||||
Source sql.NullString //
|
||||
PassGob sql.NullString //
|
||||
Ext interface{} //
|
||||
UpdateTime time.Time //
|
||||
UpdateTime sql.NullTime //
|
||||
Operator int32 //
|
||||
ErrorMsg sql.NullString //
|
||||
|
||||
lastOperator int32
|
||||
}
|
||||
|
||||
// GetPassGob Get return PassGob sql.NullString
|
||||
func (so *Source) GetPassGob() sql.NullString {
|
||||
return so.PassGob
|
||||
}
|
||||
|
||||
// SetPassGob Set PassGob sql.NullString
|
||||
func (so *Source) SetPassGob(PassGob sql.NullString) {
|
||||
so.PassGob = PassGob
|
||||
}
|
||||
|
||||
// GetLastOperator Get return lastOperator int32
|
||||
func (so *Source) GetLastOperator() int32 {
|
||||
return so.lastOperator
|
||||
}
|
||||
|
||||
// SetLastOperator Set lastOperator int32
|
||||
func (so *Source) SetLastOperator(lastOperator int32) {
|
||||
so.lastOperator = lastOperator
|
||||
}
|
||||
|
||||
// GetErrorMsg Get return ErrorMsg sql.NullString
|
||||
|
|
@ -38,12 +85,12 @@ func (so *Source) SetOperator(Operator int32) {
|
|||
}
|
||||
|
||||
// GetUpdateTime Get return UpdateTime time.Time
|
||||
func (so *Source) GetUpdateTime() time.Time {
|
||||
func (so *Source) GetUpdateTime() sql.NullTime {
|
||||
return so.UpdateTime
|
||||
}
|
||||
|
||||
// SetUpdateTime Set UpdateTime time.Time
|
||||
func (so *Source) SetUpdateTime(UpdateTime time.Time) {
|
||||
func (so *Source) SetUpdateTime(UpdateTime sql.NullTime) {
|
||||
so.UpdateTime = UpdateTime
|
||||
}
|
||||
|
||||
|
|
@ -2,48 +2,50 @@ create database if not exists `intimate_extractor`;
|
|||
use intimate_extractor;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS `anchor_info` (
|
||||
`uid` varchar(36) NOT NULL,
|
||||
`uid` bigint AUTO_INCREMENT,
|
||||
`platform` varchar(255) NOT NULL,
|
||||
`anchor_id` varchar(255) NOT NULL,
|
||||
`anchor_name` varchar(255) NOT NULL,
|
||||
`live_url` text,
|
||||
`channel` varchar(128) DEFAULT NULL,
|
||||
`show_type` varchar(255) DEFAULT NULL,
|
||||
`update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
`tags` json DEFAULT NULL,
|
||||
`ext` json DEFAULT NULL,
|
||||
`update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (`uid`),
|
||||
UNIQUE KEY `platform_anchor_id_idx` (`platform`, `anchor_id`),
|
||||
KEY `platform_idx` (`platform`),
|
||||
KEY `anchor_id_idx` (`anchor_id`),
|
||||
KEY `anchor_name_idx` (`anchor_name`),
|
||||
KEY `channel_idx` (`channel`),
|
||||
KEY `show_type_idx` (`show_type`),
|
||||
KEY `update_time_idx` (`update_time`)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS `show_log` (
|
||||
`uid` varchar(36) NOT NULL,
|
||||
CREATE TABLE IF NOT EXISTS `collect_log` (
|
||||
`uid` bigint,
|
||||
`platform` varchar(255) NOT NULL,
|
||||
`anchor_id` varchar(255) NOT NULL,
|
||||
|
||||
`is_showing` tinyint(1) DEFAULT NULL,
|
||||
`is_error` tinyint(1) DEFAULT NULL,
|
||||
`is_live_streaming` tinyint(1) DEFAULT 0,
|
||||
`is_error` tinyint(1) DEFAULT 0,
|
||||
|
||||
`followers` int(11) DEFAULT NULL,
|
||||
`views` int(11) DEFAULT NULL,
|
||||
`followers` bigint(11) DEFAULT NULL,
|
||||
`views` bigint(11) DEFAULT NULL,
|
||||
`giver` json DEFAULT NULL,
|
||||
`gratuity` int(11) DEFAULT NULL,
|
||||
`gratuity` bigint(11) DEFAULT NULL,
|
||||
|
||||
`show_title` text DEFAULT NULL,
|
||||
`show_start_time` timestamp NULL DEFAULT NULL,
|
||||
`show_end_time` timestamp NULL DEFAULT NULL,
|
||||
`update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
`live_title` text DEFAULT NULL,
|
||||
`live_start_time` timestamp NULL DEFAULT NULL,
|
||||
`live_end_time` timestamp NULL DEFAULT NULL,
|
||||
`update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
`tags` json DEFAULT NULL,
|
||||
`ext` json DEFAULT NULL,
|
||||
|
||||
`error` text DEFAULT NULL,
|
||||
`error_msg` text DEFAULT NULL,
|
||||
|
||||
KEY `uid_idx` (`uid`),
|
||||
KEY `platform_idx` (`platform`),
|
||||
KEY `anchor_id_idx` (`anchor_id`),
|
||||
KEY `is_showing_idx` (`is_showing`),
|
||||
KEY `is_live_streaming_idx` (`is_live_streaming`),
|
||||
KEY `is_error_idx` (`is_error`),
|
||||
KEY `followers_idx` (`followers`),
|
||||
KEY `views_idx` (`views`),
|
||||
|
|
|
|||
|
|
@ -7,8 +7,8 @@ CREATE TABLE IF NOT EXISTS `source_openrec` (
|
|||
`target_type` varchar(64) NOT NULL,
|
||||
`source` longtext DEFAULT NULL,
|
||||
`ext` json DEFAULT NULL,
|
||||
|
||||
`update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
`pass_gob` blob DEFAULT NULL,
|
||||
`update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
`operator` int DEFAULT 0,
|
||||
`error_msg` text DEFAULT NULL,
|
||||
PRIMARY KEY(`uid`),
|
||||
|
|
|
|||
194
store.go
194
store.go
|
|
@ -2,64 +2,47 @@ package intimate
|
|||
|
||||
import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
_ "github.com/go-sql-driver/mysql"
|
||||
)
|
||||
|
||||
// IGetSource 源接口结构
|
||||
type IGetSource interface {
|
||||
GetUid() int64 //
|
||||
GetUrl() string //
|
||||
GetTargetType() string //
|
||||
GetSource() sql.NullString //
|
||||
GetExt() interface{} //
|
||||
GetUpdateTime() time.Time //
|
||||
GetOperator() int32 //
|
||||
GetErrorMsg() sql.NullString //
|
||||
}
|
||||
|
||||
type IUpdateSource interface {
|
||||
IGetSource
|
||||
|
||||
SetExt(ext interface{}) //
|
||||
SetUpdateTime(ut time.Time) //
|
||||
SetOperator(operator int32) //
|
||||
SetErrorMsg(emsg sql.NullString) //
|
||||
}
|
||||
|
||||
// OperatorFlag 标志
|
||||
type OperatorFlag int32
|
||||
|
||||
const (
|
||||
// OperatorOK 等待被处理
|
||||
OperatorOK OperatorFlag = 100
|
||||
// OperatorExtractorOK 提取数据完成
|
||||
OperatorExtractorOK OperatorFlag = 200
|
||||
// OperatorWait 等待被处理
|
||||
OperatorWait OperatorFlag = 1000
|
||||
// OperatorError 错误标志
|
||||
OperatorError OperatorFlag = 10000
|
||||
)
|
||||
|
||||
// Store 储存
|
||||
type Store struct {
|
||||
type ISet interface {
|
||||
Set(string, interface{})
|
||||
}
|
||||
|
||||
// SourceStore 储存
|
||||
type SourceStore struct {
|
||||
table string
|
||||
db *sql.DB
|
||||
errorCount int
|
||||
errorLimit int
|
||||
}
|
||||
|
||||
// NewStore 创建一个存储实例
|
||||
func NewStore(table string) *Store {
|
||||
db, err := sql.Open("mysql", InitConfig.Database.URI)
|
||||
// NewSourceStore 创建一个存储实例
|
||||
func NewSourceStore(table string) *SourceStore {
|
||||
db, err := sql.Open("mysql", InitConfig.Database.SourceURI)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return &Store{table: table, db: db}
|
||||
return &SourceStore{table: table, db: db}
|
||||
}
|
||||
|
||||
func (store *Store) errorAlarm(err error) {
|
||||
func (store *SourceStore) errorAlarm(err error) {
|
||||
if err != nil {
|
||||
log.Println("store error: ", err)
|
||||
// 报警. 如果数据插入有问题
|
||||
|
|
@ -74,28 +57,47 @@ func (store *Store) errorAlarm(err error) {
|
|||
}
|
||||
}
|
||||
|
||||
// Insert 储存数据
|
||||
func (store *Store) Insert(isource IGetSource) {
|
||||
_, err := store.db.Exec("insert into `source_openrec`(url, target_type, source, ext, operator, error_msg) values(?,?,?,?,?,?)", isource.GetUrl(), isource.GetTargetType(), isource.GetSource(), isource.GetExt(), isource.GetOperator(), isource.GetErrorMsg())
|
||||
// Insert 插入数据
|
||||
func (store *SourceStore) Insert(isource IGetSource) {
|
||||
_, err := store.db.Exec("insert into "+store.table+"(url, target_type, source, ext, operator, error_msg) values(?,?,?,?,?,?)", isource.GetUrl(), isource.GetTargetType(), isource.GetSource(), isource.GetExt(), isource.GetOperator(), isource.GetErrorMsg())
|
||||
store.errorAlarm(err)
|
||||
}
|
||||
|
||||
// Update 储存数据
|
||||
func (store *Store) Update(isource IUpdateSource) {
|
||||
_, err := store.db.Exec("update "+store.table+" set ext = ?, operator = ?, error_msg = ? where uid = ?", isource.GetExt(), isource.GetOperator(), isource.GetErrorMsg(), isource.GetUid())
|
||||
// Update 更新数据
|
||||
func (store *SourceStore) Update(isource IUpdateSource) {
|
||||
_, err := store.db.Exec("update "+store.table+" set ext = ?, pass_gob = ?, operator = ?, error_msg = ? where uid = ?", isource.GetExt(), isource.GetPassGob(), isource.GetOperator(), isource.GetErrorMsg(), isource.GetUid())
|
||||
store.errorAlarm(err)
|
||||
}
|
||||
|
||||
// Pop 储存数据
|
||||
func (store *Store) Pop(targetType string, operators ...int32) (IUpdateSource, error) {
|
||||
// UpdateOperator 更新数据操作标志位
|
||||
func (store *SourceStore) UpdateOperator(isource IUpdateSource) {
|
||||
_, err := store.db.Exec("update "+store.table+" set operator = ?, error_msg = ? where uid = ?", isource.GetOperator(), isource.GetErrorMsg(), isource.GetUid())
|
||||
store.errorAlarm(err)
|
||||
}
|
||||
|
||||
// UpdateError 更新错误数据
|
||||
func (store *SourceStore) UpdateError(isource IUpdateSource, err error) {
|
||||
isource.SetOperator(int32(OperatorError))
|
||||
isource.SetErrorMsg(sql.NullString{String: err.Error(), Valid: true})
|
||||
_, dberr := store.db.Exec("update "+store.table+" set operator = ?, error_msg = ? where uid = ?", isource.GetOperator(), isource.GetErrorMsg(), isource.GetUid())
|
||||
store.errorAlarm(dberr)
|
||||
}
|
||||
|
||||
// Restore 恢复Operator数据状态
|
||||
func (store *SourceStore) Restore(isource IUpdateSource) {
|
||||
_, err := store.db.Exec("update "+store.table+" set operator = ? where uid = ?", isource.GetLastOperator(), isource.GetUid())
|
||||
store.errorAlarm(err)
|
||||
}
|
||||
|
||||
// Pop 弹出一条未处理的数据
|
||||
func (store *SourceStore) Pop(targetType string, operators ...int32) (IUpdateSource, error) {
|
||||
|
||||
tx, err := store.db.Begin()
|
||||
if err != nil {
|
||||
log.Println(err, targetType)
|
||||
return nil, err
|
||||
}
|
||||
var args = []interface{}{targetType}
|
||||
selectSQL := `select uid, url, target_type, source, ext, operator from ` + store.table + ` where target_type = ?`
|
||||
selectSQL := `select uid, url, target_type, source, ext, operator, update_time from ` + store.table + ` where target_type = ?`
|
||||
if len(operators) == 0 {
|
||||
selectSQL += " and operator = ?"
|
||||
args = append(args, 0)
|
||||
|
|
@ -120,21 +122,105 @@ func (store *Store) Pop(targetType string, operators ...int32) (IUpdateSource, e
|
|||
}
|
||||
}()
|
||||
|
||||
if row != nil {
|
||||
s := &Source{}
|
||||
// uid, url, target_type, source, ext, operator
|
||||
err = row.Scan(&s.Uid, &s.Url, &s.TargetType, &s.Source, &s.Ext, &s.Operator)
|
||||
if err != nil {
|
||||
log.Println(err, targetType)
|
||||
_, err = tx.Exec("update "+store.table+" set error_msg = ?, operator = ? where uid = ?", OperatorError, s.Uid)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
return nil, err
|
||||
s := &Source{}
|
||||
// uid, url, target_type, source, ext, operator
|
||||
err = row.Scan(&s.Uid, &s.Url, &s.TargetType, &s.Source, &s.Ext, &s.Operator, &s.UpdateTime)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.SetLastOperator(s.Operator)
|
||||
_, err = tx.Exec("update "+store.table+" set operator = ? where uid = ?", OperatorWait, s.Uid)
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// AnchorTable 主播表名称
|
||||
const AnchorTable string = "anchor_info"
|
||||
|
||||
// CollectLogTable 采集日志表
|
||||
const CollectLogTable string = "collect_log"
|
||||
|
||||
type ExtractorStore struct {
|
||||
db *sql.DB
|
||||
|
||||
errorCount int
|
||||
errorLimit int
|
||||
}
|
||||
|
||||
func (store *ExtractorStore) errorAlarm(err error) {
|
||||
if err != nil {
|
||||
log.Panic("store error: ", err)
|
||||
// 报警. 如果数据插入有问题
|
||||
store.errorCount++
|
||||
if store.errorCount >= store.errorLimit {
|
||||
// 数据库频繁操作初问题 报警, 减少没意义的请求
|
||||
}
|
||||
_, err = tx.Exec("update "+store.table+" set operator = ? where uid = ?", OperatorWait, s.Uid)
|
||||
return s, nil
|
||||
} else {
|
||||
if store.errorCount > 0 {
|
||||
store.errorCount--
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func NewExtractorStore() *ExtractorStore {
|
||||
db, err := sql.Open("mysql", InitConfig.Database.ExtractorURI)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return &ExtractorStore{db: db}
|
||||
}
|
||||
|
||||
/*
|
||||
`uid` bigint,
|
||||
`platform` varchar(255) NOT NULL,
|
||||
`anchor_id` varchar(255) NOT NULL,
|
||||
`anchor_name` varchar(255) NOT NULL,
|
||||
`live_url` text,
|
||||
`channel` varchar(128) DEFAULT NULL,
|
||||
`show_type` varchar(255) DEFAULT NULL,
|
||||
*/
|
||||
|
||||
// InsertAnchorInfo AnchorInfo表, 插入数据
|
||||
func (store *ExtractorStore) InsertAnchorInfo(isource IGetAnchorInfo) (Uid int64, err error) {
|
||||
// select uid from table where platform = ? and anchor_id = ?
|
||||
selectSQL := "select uid from " + AnchorTable + " where platform = ? and anchor_id = ?"
|
||||
tx, err := store.db.Begin()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
return 0, err
|
||||
}
|
||||
|
||||
return nil, errors.New("TaskQueue is nil")
|
||||
row := tx.QueryRow(selectSQL+` limit 1 for update`, isource.GetPlatform(), isource.GetAnchorId())
|
||||
|
||||
var uid int64
|
||||
if err = row.Scan(&uid); err == nil {
|
||||
return uid, nil
|
||||
}
|
||||
|
||||
result, err := tx.Exec("insert into "+AnchorTable+"(platform, anchor_id, anchor_name, live_url, channel, tags, ext) values(?,?,?,?,?,?,?);", isource.GetPlatform(), isource.GetAnchorId(), isource.GetAnchorName(), isource.GetLiveUrl(), isource.GetChannel(), isource.GetTags(), isource.GetExt())
|
||||
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
err = tx.Commit()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
err = tx.Rollback()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
return 0, err
|
||||
}
|
||||
|
||||
return result.LastInsertId()
|
||||
}
|
||||
|
||||
// InsertCollectLog CollectLog表插入数据
|
||||
func (store *ExtractorStore) InsertCollectLog(isource IGetCollectLog) error {
|
||||
_, err := store.db.Exec("insert into "+CollectLogTable+"(uid, platform, anchor_id, is_live_streaming, is_error, followers, views, giver, gratuity, live_title, live_start_time, live_end_time, update_time, tags, ext, error_msg) values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
|
||||
isource.GetUid(), isource.GetPlatform(), isource.GetAnchorId(), isource.GetIsLiveStreaming(), isource.GetIsError(), isource.GetFollowers(), isource.GetViews(), isource.GetGiver(), isource.GetGratuity(), isource.GetLiveTitle(), isource.GetLiveStartTime(), isource.GetLiveEndTime(), isource.GetUpdateTime(), isource.GetTags(), isource.GetExt(), isource.GetErrorMsg(),
|
||||
)
|
||||
store.errorAlarm(err)
|
||||
return err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -26,8 +26,8 @@ func TestStoreInsertCase1(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestStorePopCase1(t *testing.T) {
|
||||
store := NewStore("source_openrec")
|
||||
source, err := store.Pop("openrec_ranking")
|
||||
store := NewSourceStore("source_openrec")
|
||||
source, err := store.Pop(string(TTOpenrecRanking))
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
|
|
|||
10
table_list.go
Normal file
10
table_list.go
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
package intimate
|
||||
|
||||
// SourceTable 源的table列表
|
||||
type SourceTable string
|
||||
|
||||
const (
|
||||
// STOpenrec openrec源table名称
|
||||
STOpenrec SourceTable = "source_openrec"
|
||||
)
|
||||
|
||||
12
target_type_list.go
Normal file
12
target_type_list.go
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
package intimate
|
||||
|
||||
// TargetType 源的 目标类型 列表
|
||||
type TargetType string
|
||||
|
||||
const (
|
||||
// TTOpenrecRanking openrec源TargetType名称
|
||||
TTOpenrecRanking TargetType = "openrec_ranking"
|
||||
|
||||
// TTOpenrecUser openrec源TargetType名称
|
||||
TTOpenrecUser TargetType = "openrec_ranking"
|
||||
)
|
||||
3
tasks/openrec/openrec_task1/.gitignore
vendored
3
tasks/openrec/openrec_task1/.gitignore
vendored
|
|
@ -1 +1,2 @@
|
|||
openrec_task1
|
||||
openrec_task1
|
||||
log
|
||||
|
|
@ -1 +0,0 @@
|
|||
../../../config.yaml
|
||||
|
|
@ -29,8 +29,8 @@ func (or *OpenrecRankingTest) Execute(cxt *hunter.TaskContext) {
|
|||
t.Error("rank is error. result raw is ", result.Raw)
|
||||
}
|
||||
|
||||
if cxt.Workflow().GetQuery().Get("page") != "1" {
|
||||
t.Error("workflow page error")
|
||||
if cxt.Temporary().GetQuery().Get("page") != "1" {
|
||||
t.Error("Temporary page error")
|
||||
}
|
||||
// t.Error(string(resp.Content()))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,19 +4,21 @@ import (
|
|||
"database/sql"
|
||||
"intimate"
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strconv"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/474420502/hunter"
|
||||
"github.com/tidwall/gjson"
|
||||
)
|
||||
|
||||
var targetTypeRanking = "openrec_ranking"
|
||||
var targetTypeUser = "openrec_user"
|
||||
var openrecRanking *OpenrecRanking
|
||||
|
||||
// store 源存储实例, 为存储源数据的实现. 表格具体参考sql/intimate_source.sql
|
||||
var store *intimate.Store = intimate.NewStore("source_openrec")
|
||||
var store *intimate.SourceStore = intimate.NewSourceStore(string(intimate.STOpenrec))
|
||||
|
||||
func init() {
|
||||
|
||||
|
|
@ -44,7 +46,16 @@ type OpenrecRanking struct {
|
|||
// Execute 执行方法
|
||||
func (or *OpenrecRanking) Execute(cxt *hunter.TaskContext) {
|
||||
|
||||
for {
|
||||
var loop int32 = 1
|
||||
|
||||
go func() {
|
||||
signalchan := make(chan os.Signal)
|
||||
signal.Notify(signalchan, syscall.SIGKILL, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGSTOP)
|
||||
log.Println("accept stop command:", <-signalchan)
|
||||
atomic.StoreInt32(&loop, 0)
|
||||
}()
|
||||
|
||||
for atomic.LoadInt32(&loop) > 0 {
|
||||
|
||||
resp, err := cxt.Hunt()
|
||||
if err != nil {
|
||||
|
|
@ -52,7 +63,7 @@ func (or *OpenrecRanking) Execute(cxt *hunter.TaskContext) {
|
|||
break
|
||||
}
|
||||
|
||||
wf := cxt.Workflow()
|
||||
wf := cxt.Temporary()
|
||||
|
||||
content := resp.Content()
|
||||
if len(content) <= 200 {
|
||||
|
|
@ -67,7 +78,7 @@ func (or *OpenrecRanking) Execute(cxt *hunter.TaskContext) {
|
|||
|
||||
data.SetSource(sql.NullString{String: userid, Valid: len(userid) > 0})
|
||||
data.SetUrl(wf.GetRawURL())
|
||||
data.SetTargetType(targetTypeUser)
|
||||
data.SetTargetType(string(intimate.TTOpenrecUser))
|
||||
store.Insert(data)
|
||||
}
|
||||
}
|
||||
|
|
@ -78,7 +89,7 @@ func (or *OpenrecRanking) Execute(cxt *hunter.TaskContext) {
|
|||
log.Println(err)
|
||||
return
|
||||
}
|
||||
|
||||
return
|
||||
page++
|
||||
querys.Set("page", strconv.Itoa(page))
|
||||
wf.SetQuery(querys)
|
||||
|
|
|
|||
|
|
@ -1,2 +0,0 @@
|
|||
database:
|
||||
uri: "root:@tcp(127.0.0.1:4000)/intimate_source"
|
||||
|
|
@ -5,17 +5,23 @@ import (
|
|||
"encoding/json"
|
||||
"intimate"
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strconv"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/474420502/gcurl"
|
||||
"github.com/tidwall/gjson"
|
||||
|
||||
"github.com/474420502/hunter"
|
||||
)
|
||||
|
||||
var targetTypeUser = "openrec_user"
|
||||
var targetTypeRanking = "openrec_ranking"
|
||||
var oer *OpenrecExtratorRanking
|
||||
|
||||
// store 源存储实例, 为存储源数据的实现. 表格具体参考sql/intimate_source.sql
|
||||
var store *intimate.Store = intimate.NewStore("source_openrec")
|
||||
var store *intimate.SourceStore = intimate.NewSourceStore(string(intimate.STOpenrec))
|
||||
|
||||
func init() {
|
||||
oer = &OpenrecExtratorRanking{}
|
||||
|
|
@ -29,16 +35,23 @@ type OpenrecExtratorRanking struct {
|
|||
// Execute 执行方法
|
||||
func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) {
|
||||
|
||||
for {
|
||||
var loop int32 = 1
|
||||
|
||||
source, err := store.Pop(targetTypeUser)
|
||||
if err != nil {
|
||||
go func() {
|
||||
signalchan := make(chan os.Signal)
|
||||
signal.Notify(signalchan, syscall.SIGKILL, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGSTOP)
|
||||
log.Println("accept stop command:", <-signalchan)
|
||||
atomic.StoreInt32(&loop, 0)
|
||||
}()
|
||||
|
||||
for atomic.LoadInt32(&loop) > 0 {
|
||||
|
||||
source, err := store.Pop(string(intimate.TTOpenrecUser))
|
||||
|
||||
if source == nil || err != nil {
|
||||
log.Println(err)
|
||||
return
|
||||
}
|
||||
|
||||
if source == nil {
|
||||
return
|
||||
time.Sleep(time.Second * 2)
|
||||
continue
|
||||
}
|
||||
|
||||
userSource := &intimate.Source{}
|
||||
|
|
@ -48,36 +61,81 @@ func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) {
|
|||
|
||||
wf := cxt.Session().Get(userUrl)
|
||||
resp, err := wf.Execute()
|
||||
source.SetUpdateTime(time.Now())
|
||||
source.SetUpdateTime(sql.NullTime{Time: time.Now(), Valid: true})
|
||||
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
|
||||
source.SetOperator(int32(intimate.OperatorError))
|
||||
source.SetErrorMsg(sql.NullString{String: err.Error(), Valid: true})
|
||||
store.UpdateError(source, err)
|
||||
continue
|
||||
}
|
||||
|
||||
cookies := cxt.Session().GetCookies(wf.GetParsedURL())
|
||||
|
||||
scurl := "https://www.openrec.tv/viewapp/api/v6/supporters?identify_id=sumomo_xqx&month=&Uuid=B96EE988-E3A2-4A44-A543-611A8B4BC683&Token=46598c320408bd69ae3c63298f6f4a3a97354175&Random=AZVXNAAXQVMOSVWNDPIQ&page_number=1 -H 'accept: application/json, text/javascript, */*; q=0.01' -H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36' -H 'cookie: uuid=B96EE988-E3A2-4A44-A543-611A8B4BC683;' --compressed"
|
||||
curl := gcurl.ParseRawCURL(scurl)
|
||||
supportersSession := curl.CreateSession()
|
||||
|
||||
temporary := curl.CreateTemporary(supportersSession)
|
||||
supportersSession.SetCookies(temporary.GetParsedURL(), cookies)
|
||||
var supporters []string
|
||||
for {
|
||||
|
||||
supportersQuery := temporary.GetQuery()
|
||||
|
||||
for _, cookie := range cookies {
|
||||
if cookie.Name == "uuid" {
|
||||
supportersQuery.Set("Uuid", cookie.Value)
|
||||
continue
|
||||
}
|
||||
|
||||
if cookie.Name == "token" {
|
||||
supportersQuery.Set("Token", cookie.Value)
|
||||
continue
|
||||
}
|
||||
|
||||
if cookie.Name == "random" {
|
||||
supportersQuery.Set("Random", cookie.Value)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
supportersQuery.Set("identify_id", source.GetSource().String)
|
||||
temporary.SetQuery(supportersQuery)
|
||||
|
||||
resp, err := temporary.Execute()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
supporterjson := gjson.ParseBytes(resp.Content())
|
||||
supporterdata := supporterjson.Get("data")
|
||||
if supporterdata.Type == gjson.Null {
|
||||
break
|
||||
}
|
||||
supporters = append(supporters, string(resp.Content()))
|
||||
|
||||
page := supportersQuery.Get("page_number")
|
||||
pageint, err := strconv.Atoi(page)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
break
|
||||
}
|
||||
pageint++
|
||||
page = strconv.Itoa(pageint)
|
||||
supportersQuery.Set("page_number", page)
|
||||
temporary.SetQuery(supportersQuery)
|
||||
}
|
||||
|
||||
// cookies := cxt.Session().GetCookies(wf.GetParsedURL())
|
||||
ext := make(map[string]interface{})
|
||||
|
||||
ext["supporters"] = supporters
|
||||
ext["user"] = string(resp.Content())
|
||||
|
||||
wf = cxt.Session().Get(userUrl + "/supporters")
|
||||
resp, err = wf.Execute()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
source.SetOperator(int32(intimate.OperatorError))
|
||||
source.SetErrorMsg(sql.NullString{String: err.Error(), Valid: true})
|
||||
continue
|
||||
}
|
||||
ext["user_supporters"] = string(resp.Content())
|
||||
|
||||
wf = cxt.Session().Get("https://www.openrec.tv/live/" + userid)
|
||||
resp, err = wf.Execute()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
source.SetOperator(int32(intimate.OperatorError))
|
||||
source.SetErrorMsg(sql.NullString{String: err.Error(), Valid: true})
|
||||
store.UpdateError(source, err)
|
||||
continue
|
||||
}
|
||||
ext["user_live"] = string(resp.Content())
|
||||
|
|
@ -85,15 +143,13 @@ func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) {
|
|||
extJsonBytes, err := json.Marshal(ext)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
source.SetOperator(int32(intimate.OperatorError))
|
||||
source.SetErrorMsg(sql.NullString{String: err.Error(), Valid: true})
|
||||
store.UpdateError(source, err)
|
||||
continue
|
||||
}
|
||||
|
||||
source.SetOperator(int32(intimate.OperatorOK))
|
||||
source.SetExt(string(extJsonBytes))
|
||||
store.Update(source)
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
235
testfile/openrec_supporter.json
Normal file
235
testfile/openrec_supporter.json
Normal file
|
|
@ -0,0 +1,235 @@
|
|||
{
|
||||
"status": 0,
|
||||
"data": {
|
||||
"items": [
|
||||
{
|
||||
"user_id": 655750535,
|
||||
"user_name": "\u8471(\u306d\u304e)\u3093\u3061\u3087",
|
||||
"user_icon": null,
|
||||
"user_key": "NeginCho",
|
||||
"user_bg": "https:\/\/www.openrec.tv\/viewapp\/images\/v8\/img_back.png",
|
||||
"background_file": "https:\/\/www.openrec.tv\/viewapp\/images\/v8\/img_back.png",
|
||||
"user_introduce": "",
|
||||
"user_type": 2,
|
||||
"identify_id": "ebev00Nwej3",
|
||||
"followed_flg": 0,
|
||||
"movie_total_views": null,
|
||||
"onair_flg": 0,
|
||||
"onair": null,
|
||||
"update_at": null,
|
||||
"total_yells": 480,
|
||||
"yell_image_url": "https:\/\/dqd0jw5gvbchn.cloudfront.net\/yell\/22\/2e6d5b4895ae00b656a700e2d25ef41e9e2600cd.gif",
|
||||
"supporter_rank": 37
|
||||
},
|
||||
{
|
||||
"user_id": 97105260,
|
||||
"user_name": "\u5f71\u306c\u3044",
|
||||
"user_icon": "https:\/\/openrec-appdata.s3.amazonaws.com\/user\/971053\/97105260.png?1496661867",
|
||||
"user_key": "touka1308",
|
||||
"user_bg": "https:\/\/www.openrec.tv\/viewapp\/images\/v8\/img_back.png",
|
||||
"background_file": "https:\/\/www.openrec.tv\/viewapp\/images\/v8\/img_back.png",
|
||||
"user_introduce": "",
|
||||
"user_type": 2,
|
||||
"identify_id": "PDZBatTy0sDvB",
|
||||
"followed_flg": 0,
|
||||
"movie_total_views": null,
|
||||
"onair_flg": 0,
|
||||
"onair": null,
|
||||
"update_at": null,
|
||||
"total_yells": 480,
|
||||
"yell_image_url": "https:\/\/dqd0jw5gvbchn.cloudfront.net\/yell\/22\/2e6d5b4895ae00b656a700e2d25ef41e9e2600cd.gif",
|
||||
"supporter_rank": 38
|
||||
},
|
||||
{
|
||||
"user_id": 45079954,
|
||||
"user_name": "\u3086\u3063\u3061\u3083",
|
||||
"user_icon": "https:\/\/openrec-appdata.s3.amazonaws.com\/user\/450800\/45079954.png?1482929097",
|
||||
"user_key": "yuccha1444",
|
||||
"user_bg": "https:\/\/www.openrec.tv\/viewapp\/images\/v8\/img_back.png",
|
||||
"background_file": "https:\/\/www.openrec.tv\/viewapp\/images\/v8\/img_back.png",
|
||||
"user_introduce": "PUBG\u3000\u30b9\u30d7\u30e9\u30c8\u30a5\u30fc\u30f3\u30e1\u30a4\u30f3\u3067\u3059\u3002\n",
|
||||
"user_type": 2,
|
||||
"identify_id": "UQIKagVQXfbRX",
|
||||
"followed_flg": 0,
|
||||
"movie_total_views": "466",
|
||||
"onair_flg": 0,
|
||||
"onair": null,
|
||||
"update_at": "2020\/7\/12 14:36:11",
|
||||
"total_yells": 480,
|
||||
"yell_image_url": "https:\/\/dqd0jw5gvbchn.cloudfront.net\/yell\/22\/2e6d5b4895ae00b656a700e2d25ef41e9e2600cd.gif",
|
||||
"supporter_rank": 39
|
||||
},
|
||||
{
|
||||
"user_id": 72882108,
|
||||
"user_name": "\u3042\u308b\u3050\u308c",
|
||||
"user_icon": "https:\/\/openrec-appdata.s3.amazonaws.com\/user\/728822\/72882108.png?1502904526",
|
||||
"user_key": "foarsiljw10",
|
||||
"user_bg": "https:\/\/www.openrec.tv\/viewapp\/images\/v8\/img_back.png",
|
||||
"background_file": "https:\/\/www.openrec.tv\/viewapp\/images\/v8\/img_back.png",
|
||||
"user_introduce": "",
|
||||
"user_type": 2,
|
||||
"identify_id": "O2d6Htfxc8PvB",
|
||||
"followed_flg": 0,
|
||||
"movie_total_views": null,
|
||||
"onair_flg": 0,
|
||||
"onair": null,
|
||||
"update_at": null,
|
||||
"total_yells": 480,
|
||||
"yell_image_url": "https:\/\/dqd0jw5gvbchn.cloudfront.net\/yell\/22\/2e6d5b4895ae00b656a700e2d25ef41e9e2600cd.gif",
|
||||
"supporter_rank": 40
|
||||
},
|
||||
{
|
||||
"user_id": 709331194,
|
||||
"user_name": "\u3044\u3044\u3093\u3059\u304b240\u30a8\u30fc\u30eb\uff01\u3042\u3056\u3059\uff01",
|
||||
"user_icon": null,
|
||||
"user_key": "oriton",
|
||||
"user_bg": "https:\/\/www.openrec.tv\/viewapp\/images\/v8\/img_back.png",
|
||||
"background_file": "https:\/\/www.openrec.tv\/viewapp\/images\/v8\/img_back.png",
|
||||
"user_introduce": "",
|
||||
"user_type": 2,
|
||||
"identify_id": "b8JQo5PvZAJ",
|
||||
"followed_flg": 0,
|
||||
"movie_total_views": null,
|
||||
"onair_flg": 0,
|
||||
"onair": null,
|
||||
"update_at": null,
|
||||
"total_yells": 480,
|
||||
"yell_image_url": "https:\/\/dqd0jw5gvbchn.cloudfront.net\/yell\/22\/2e6d5b4895ae00b656a700e2d25ef41e9e2600cd.gif",
|
||||
"supporter_rank": 41
|
||||
},
|
||||
{
|
||||
"user_id": 3756302,
|
||||
"user_name": "\u3048\u3082\u3042\u3044\u3046",
|
||||
"user_icon": "https:\/\/openrec-appdata.s3.amazonaws.com\/user\/37564\/3756302.png?1457963374",
|
||||
"user_key": "nininiku",
|
||||
"user_bg": "https:\/\/www.openrec.tv\/viewapp\/images\/v8\/img_back.png",
|
||||
"background_file": "https:\/\/www.openrec.tv\/viewapp\/images\/v8\/img_back.png",
|
||||
"user_introduce": "",
|
||||
"user_type": 2,
|
||||
"identify_id": "etB8pDT5kl2Gs",
|
||||
"followed_flg": 0,
|
||||
"movie_total_views": null,
|
||||
"onair_flg": 0,
|
||||
"onair": null,
|
||||
"update_at": null,
|
||||
"total_yells": 400,
|
||||
"yell_image_url": "https:\/\/dqd0jw5gvbchn.cloudfront.net\/yell\/22\/2e6d5b4895ae00b656a700e2d25ef41e9e2600cd.gif",
|
||||
"supporter_rank": 42
|
||||
},
|
||||
{
|
||||
"user_id": 102189594,
|
||||
"user_name": "K",
|
||||
"user_icon": "https:\/\/openrec-appdata.s3.amazonaws.com\/user\/1021896\/102189594.png?1589000488",
|
||||
"user_key": "sc__pk",
|
||||
"user_bg": "https:\/\/www.openrec.tv\/viewapp\/images\/v8\/img_back.png",
|
||||
"background_file": "https:\/\/www.openrec.tv\/viewapp\/images\/v8\/img_back.png",
|
||||
"user_introduce": "",
|
||||
"user_type": 2,
|
||||
"identify_id": "FejYFKk3wcoGV",
|
||||
"followed_flg": 0,
|
||||
"movie_total_views": null,
|
||||
"onair_flg": 0,
|
||||
"onair": null,
|
||||
"update_at": null,
|
||||
"total_yells": 400,
|
||||
"yell_image_url": "https:\/\/dqd0jw5gvbchn.cloudfront.net\/yell\/22\/2e6d5b4895ae00b656a700e2d25ef41e9e2600cd.gif",
|
||||
"supporter_rank": 43
|
||||
},
|
||||
{
|
||||
"user_id": 193073369,
|
||||
"user_name": "minomushi",
|
||||
"user_icon": "https:\/\/openrec-appdata.s3.amazonaws.com\/user\/1930734\/193073369.png?1519733979",
|
||||
"user_key": "daitaidaitai",
|
||||
"user_bg": "https:\/\/www.openrec.tv\/viewapp\/images\/v8\/img_back.png",
|
||||
"background_file": "https:\/\/www.openrec.tv\/viewapp\/images\/v8\/img_back.png",
|
||||
"user_introduce": "",
|
||||
"user_type": 2,
|
||||
"identify_id": "v_QBD_XBpWoAS",
|
||||
"followed_flg": 0,
|
||||
"movie_total_views": null,
|
||||
"onair_flg": 0,
|
||||
"onair": null,
|
||||
"update_at": null,
|
||||
"total_yells": 400,
|
||||
"yell_image_url": "https:\/\/dqd0jw5gvbchn.cloudfront.net\/yell\/22\/2e6d5b4895ae00b656a700e2d25ef41e9e2600cd.gif",
|
||||
"supporter_rank": 44
|
||||
},
|
||||
{
|
||||
"user_id": 126454208,
|
||||
"user_name": "\u7d76\u671b\u304a\u3063\u3071\u3043\u3061\u3083\u3093",
|
||||
"user_icon": "https:\/\/openrec-appdata.s3.amazonaws.com\/user\/1264543\/126454208.png?1575873318",
|
||||
"user_key": "chiyongo",
|
||||
"user_bg": "https:\/\/openrec-appdata.s3.amazonaws.com\/user_background\/1264543\/126454208_cover.png?1557337359",
|
||||
"background_file": "https:\/\/openrec-appdata.s3.amazonaws.com\/user_background\/1264543\/126454208_cover.png?1557337359",
|
||||
"user_introduce": "\u56fd\u6307\u5b9a\u306e\u96e3\u75c5\u3068\u6226\u3063\u3066\u3044\u307e\u3059\u3002\n\u3067\u3082\u6c17\u6301\u3061\u7684\u306b\u306f\u6bce\u65e5\u5143\u6c17\uff01\uff01\uff01\n\u6bce\u65e5\u30b2\u30fc\u30e0\u3059\u308b\u6642\u9593\u304c\u8db3\u308a\u306a\u3044\u60b2\u3057\u307f\u3002\nTwitter\u219288tico88",
|
||||
"user_type": 2,
|
||||
"identify_id": "qWbsDP_t1WjER",
|
||||
"followed_flg": 0,
|
||||
"movie_total_views": null,
|
||||
"onair_flg": 0,
|
||||
"onair": null,
|
||||
"update_at": null,
|
||||
"total_yells": 400,
|
||||
"yell_image_url": "https:\/\/dqd0jw5gvbchn.cloudfront.net\/yell\/22\/2e6d5b4895ae00b656a700e2d25ef41e9e2600cd.gif",
|
||||
"supporter_rank": 45
|
||||
},
|
||||
{
|
||||
"user_id": 608946713,
|
||||
"user_name": "\u91ce\u826f\u30aa\u30af\u30bf\u30f3",
|
||||
"user_icon": null,
|
||||
"user_key": "ryryryryryryo",
|
||||
"user_bg": "https:\/\/www.openrec.tv\/viewapp\/images\/v8\/img_back.png",
|
||||
"background_file": "https:\/\/www.openrec.tv\/viewapp\/images\/v8\/img_back.png",
|
||||
"user_introduce": "",
|
||||
"user_type": 2,
|
||||
"identify_id": "jAnaeQ11WOg",
|
||||
"followed_flg": 0,
|
||||
"movie_total_views": null,
|
||||
"onair_flg": 0,
|
||||
"onair": null,
|
||||
"update_at": null,
|
||||
"total_yells": 400,
|
||||
"yell_image_url": "https:\/\/dqd0jw5gvbchn.cloudfront.net\/yell\/22\/2e6d5b4895ae00b656a700e2d25ef41e9e2600cd.gif",
|
||||
"supporter_rank": 46
|
||||
},
|
||||
{
|
||||
"user_id": 34933103,
|
||||
"user_name": "\u306f\u308b\u3061",
|
||||
"user_icon": "https:\/\/openrec-appdata.s3.amazonaws.com\/user\/349332\/34933103.png?1580674220",
|
||||
"user_key": "renaryu07131003",
|
||||
"user_bg": "https:\/\/openrec-appdata.s3.amazonaws.com\/user_background\/349332\/34933103_cover.png?1580674220",
|
||||
"background_file": "https:\/\/openrec-appdata.s3.amazonaws.com\/user_background\/349332\/34933103_cover.png?1580674220",
|
||||
"user_introduce": "\u3053\u3093\u306b\u3061\u306f\u3002",
|
||||
"user_type": 2,
|
||||
"identify_id": "idqwub_qCyhxd",
|
||||
"followed_flg": 0,
|
||||
"movie_total_views": null,
|
||||
"onair_flg": 0,
|
||||
"onair": null,
|
||||
"update_at": null,
|
||||
"total_yells": 400,
|
||||
"yell_image_url": "https:\/\/dqd0jw5gvbchn.cloudfront.net\/yell\/22\/2e6d5b4895ae00b656a700e2d25ef41e9e2600cd.gif",
|
||||
"supporter_rank": 47
|
||||
},
|
||||
{
|
||||
"user_id": 651299052,
|
||||
"user_name": "\u3084\u3080\u304f\u3093",
|
||||
"user_icon": "https:\/\/openrec-appdata.s3.amazonaws.com\/user\/6512991\/651299052.png?1581381215",
|
||||
"user_key": "kangsangsoo",
|
||||
"user_bg": "https:\/\/www.openrec.tv\/viewapp\/images\/v8\/img_back.png",
|
||||
"background_file": "https:\/\/www.openrec.tv\/viewapp\/images\/v8\/img_back.png",
|
||||
"user_introduce": "",
|
||||
"user_type": 2,
|
||||
"identify_id": "3XvQmvpxLgL",
|
||||
"followed_flg": 0,
|
||||
"movie_total_views": null,
|
||||
"onair_flg": 0,
|
||||
"onair": null,
|
||||
"update_at": null,
|
||||
"total_yells": 400,
|
||||
"yell_image_url": "https:\/\/dqd0jw5gvbchn.cloudfront.net\/yell\/22\/2e6d5b4895ae00b656a700e2d25ef41e9e2600cd.gif",
|
||||
"supporter_rank": 48
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
1659
testfile/openrec_user.html
Executable file
1659
testfile/openrec_user.html
Executable file
File diff suppressed because it is too large
Load Diff
392
testfile/openrec_user_live.html
Executable file
392
testfile/openrec_user_live.html
Executable file
File diff suppressed because one or more lines are too long
7
testfile/openrec_user_supporters.json
Executable file
7
testfile/openrec_user_supporters.json
Executable file
File diff suppressed because one or more lines are too long
44
utils.go
Normal file
44
utils.go
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
package intimate
|
||||
|
||||
import (
|
||||
"log"
|
||||
"time"
|
||||
)
|
||||
|
||||
var zeroTime time.Time
|
||||
|
||||
func init() {
|
||||
|
||||
tm, err := time.Parse("15:04:05", "0:00:00")
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
zeroTime = tm
|
||||
|
||||
}
|
||||
|
||||
// ParseDuration time to duration eg: 1:40:00 -> time.Duration
|
||||
func ParseDuration(dt string) (time.Duration, error) {
|
||||
|
||||
var parse []byte = []byte("00:00:00")
|
||||
|
||||
j := len(parse) - 1
|
||||
for i := len(dt) - 1; i >= 0; i-- {
|
||||
c := dt[i]
|
||||
if c != ':' {
|
||||
parse[j] = dt[i]
|
||||
} else {
|
||||
for parse[j] != ':' {
|
||||
j--
|
||||
}
|
||||
}
|
||||
j--
|
||||
}
|
||||
|
||||
tdt, err := time.Parse("15:04:05", string(parse))
|
||||
if err != nil {
|
||||
|
||||
return time.Duration(0), err
|
||||
}
|
||||
return tdt.Sub(zeroTime), nil
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user