test/intimate
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

add new nimo code

Browse Source 
change more
This commit is contained in:
eson
2020-08-28 19:07:12 +08:00
parent eee4e30585
commit 5d50119825
30 changed files with 10540 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
extractor/openrec_extractor/openrec_extractor.go
View File

@@ -27,7 +27,7 @@ type UserLive struct {
Title string `exp:"//h1[contains(@class,'MovieTitle__Title')]"`
LiveStartTime string `exp:"//meta[@itemprop='uploadDate']/@content"`
LiveEndTime string `exp:"//meta[@itemprop='duration']/@content"`
Tags []string `exp:"//a[contains(@class,'TagButton')]"`
Tags []string `exp:"//div[contains(@class,'MovieMetaContent__TagContainer')]//a[@role ='button']"`
}
// Execute 执行
@@ -135,8 +135,9 @@ func Execute() {
streamer.Uid = source.StreamerId.Int64
streamer.UpdateTime = source.UpdateTime
streamer.Tags = clog.Tags
if clog.Tags != nil {
streamer.Tags = clog.Tags
}
clog.Platform = intimate.Popenrec
clog.UserId = userId
clog.UpdateTime = source.UpdateTime

36
extractor/twitcasting_extractor/twitcasting_extractor.go
View File

@@ -113,42 +113,38 @@ func main() {
clog.LiveTitle = sql.NullString{String: ldata.LiveTitle, Valid: true}
clog.Tags = tags
// fl, err := intimate.ParseNumberEx(ldata.Follower)
// if err == nil {
clog.Followers = sql.NullInt64{Int64: int64(ldata.Follower), Valid: true}
switch {
case ldata.Follower <= 100:
streamer.UpdateInterval = 360
streamer.UpdateInterval = 720
case ldata.Follower <= 1000:
streamer.UpdateInterval = 320
case ldata.Follower <= 10000:
streamer.UpdateInterval = 240
case ldata.Follower <= 100:
streamer.UpdateInterval = 120
default:
streamer.UpdateInterval = 60
streamer.UpdateInterval = 120
}
streamer.UpdateInterval = 60
// } else {
// log.Println(err)
// }
clog.Views = sql.NullInt64{Int64: ldata.MaxViews, Valid: true}
// views, err := strconv.Atoi(ldata.MaxViews)
// if err == nil {
// clog.Views = sql.NullInt64{Int64: int64(views), Valid: true}
// } else {
// clog.Views = sql.NullInt64{Int64: int64(0), Valid: true}
// // log.Println(err, streamer.UserId)
// }
// st, err := strconv.Atoi(ldata.LiveStart)
if ldata.LiveStart != "" {
st, err := time.Parse("Mon, 02 Jan 2006 15:04:05 -0700", ldata.LiveStart)
if err == nil {
startTime := st
clog.LiveStartTime = sql.NullTime{Time: startTime, Valid: true}
dt, err := strconv.Atoi(ldata.LiveDuration)
if time.Now().Sub(startTime) >= time.Hour*24*90 {
liveduration := time.Now().Sub(startTime)
switch {
case liveduration >= time.Hour*24*240:
streamer.Operator = 5
case liveduration >= time.Hour*24*60:
streamer.UpdateInterval = 60 * 24 * 30
case liveduration >= time.Hour*24*30:
streamer.UpdateInterval = 60 * 24 * 15
case liveduration >= time.Hour*24*15:
streamer.UpdateInterval = 60 * 24 * 7
case liveduration >= time.Hour*24*7:
streamer.UpdateInterval = 60 * 24 * 3
}
if err == nil {

28
extractor/twitch_extractor/tiwtch_extractor.go
View File

@@ -22,21 +22,14 @@ func main() {
wd := intimate.GetChromeDriver(3030)
ps := intimate.NewPerfectShutdown()
counter := intimate.NewCounter()
counter.SetMaxLimit(200)
counter.SetMaxToDo(func(olist ...interface{}) error {
owd := olist[0].(*selenium.WebDriver)
(*owd).Close()
(*owd).Quit()
*owd = intimate.GetChromeDriver(3030)
return nil
}, &wd)
var count = 0
var countlimt = 200
var lasterr error = nil
// var err error
for !ps.IsClose() {
streamer, err := estore.Pop(intimate.Ptwitch, 0)
streamer, err := estore.PopNoWait(intimate.Ptwitch, "tags is NULL", 15)
if streamer == nil || err != nil {
if err != lasterr {
log.Println(err, lasterr)
@@ -117,18 +110,23 @@ func main() {
case fl > 100000:
streamer.UpdateInterval = 120
case fl > 10000:
streamer.UpdateInterval = 240
streamer.UpdateInterval = 240 * 2
case fl > 1000:
streamer.UpdateInterval = 360
streamer.UpdateInterval = 360 * 2
case fl > 100:
streamer.UpdateInterval = 720
streamer.UpdateInterval = 720 * 2
case fl > 0:
streamer.UpdateInterval = 1440
streamer.UpdateInterval = 1440 * 4
}
streamer.UpdateTime = clog.UpdateTime
estore.UpdateStreamer(streamer)
counter.AddWithReset(1)
count++
if count >= countlimt {
count = 0
// wd.Quit()
wd = intimate.GetChromeDriver(3030)
}
}
wd.Close()