test/intimate
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

完成openrec源的获取.

Browse Source
This commit is contained in:
eson 2020-07-09 17:09:46 +08:00
parent a027861f5a
commit 2e9a803645
5 changed files with 66 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -4,5 +4,6 @@ screenlog.*
*.7z
intimate
*.gz
debug.test

2
sql/intimate_extractor.sql
View File

@ -6,7 +6,7 @@ CREATE TABLE IF NOT EXISTS `anchor_info` (
`platform` varchar(255) NOT NULL,
`anchor_id` varchar(255) NOT NULL,
`anchor_name` varchar(255) NOT NULL,
`platform_url` text NOT NULL,
`live_url` text,
`channel` varchar(128) DEFAULT NULL,
`show_type` varchar(255) DEFAULT NULL,
`update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,

29
tasks/openrec/openrec_task1/task_openrec.go
View File

@ -8,9 +8,11 @@ import (
"time"
"github.com/474420502/hunter"
"github.com/tidwall/gjson"
)
var targetTypeRanking = "openrec_ranking"
var targetTypeUser = "openrec_user"
var openrecRanking *OpenrecRanking
// store 源存储实例, 为存储源数据的实现. 表格具体参考sql/intimate_source.sql
@ -44,7 +46,6 @@ func (or *OpenrecRanking) Execute(cxt *hunter.TaskContext) {
for {
errorMsg := sql.NullString{Valid: false}
resp, err := cxt.Hunt()
if err != nil {
log.Println(err)
@ -53,36 +54,34 @@ func (or *OpenrecRanking) Execute(cxt *hunter.TaskContext) {
wf := cxt.Workflow()
data := &intimate.Source{}
content := resp.Content()
if len(content) <= 200 {
return
}
data.SetSource(sql.NullString{String: string(content), Valid: len(content) > 0})
data.SetUrl(wf.GetRawURL())
data.SetTargetType(targetTypeRanking)
result := gjson.ParseBytes(content)
if result.IsArray() {
for _, User := range result.Array() {
data := &intimate.Source{}
userid := User.Get("channel.id").String()
data.SetSource(sql.NullString{String: userid, Valid: len(userid) > 0})
data.SetUrl(wf.GetRawURL())
data.SetTargetType(targetTypeUser)
store.Insert(data)
}
}
querys := wf.GetQuery()
page, err := strconv.Atoi(querys.Get("page"))
if err != nil {
log.Println(err)
errorMsg.String = err.Error()
errorMsg.Valid = true
data.SetErrorMsg(errorMsg)
data.SetOperator(10000)
store.Insert(data)
return
}
page++
querys.Set("page", strconv.Itoa(page))
wf.SetQuery(querys)
data.SetErrorMsg(errorMsg)
store.Insert(data)
time.Sleep(time.Second * 2)
}
}

94
tasks/openrec/openrec_task2/task_openrec.go
View File

@ -8,7 +8,6 @@ import (
"time"
"github.com/474420502/hunter"
"github.com/tidwall/gjson"
)
var targetTypeUser = "openrec_user"
@ -32,7 +31,7 @@ func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) {
for {
source, err := store.Pop(targetTypeRanking)
source, err := store.Pop(targetTypeUser)
if err != nil {
log.Println(err)
return
@ -42,52 +41,59 @@ func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) {
return
}
result := gjson.Parse(source.GetSource().String)
if result.IsArray() {
for _, User := range result.Array() {
userid := User.Get("channel.id").String()
ext := make(map[string]interface{})
userSource := &intimate.Source{}
userid := source.GetSource().String
userUrl := "https://www.openrec.tv/user/" + userid
userSource.SetUrl(userUrl)
wf := cxt.Session().Get("https://www.openrec.tv/user/" + userid)
resp, err := wf.Execute()
source.SetUpdateTime(time.Now())
wf := cxt.Session().Get(userUrl)
resp, err := wf.Execute()
source.SetUpdateTime(time.Now())
if err != nil {
log.Println(err)
if err != nil {
log.Println(err)
source.SetOperator(int32(intimate.OperatorError))
source.SetErrorMsg(sql.NullString{String: err.Error(), Valid: true})
continue
}
ext["user"] = string(resp.Content())
wf = cxt.Session().Get("https://www.openrec.tv/user/" + userid + "/supporters")
resp, err = wf.Execute()
if err != nil {
log.Println(err)
source.SetOperator(int32(intimate.OperatorError))
source.SetErrorMsg(sql.NullString{String: err.Error(), Valid: true})
continue
}
ext["user_supporters"] = string(resp.Content())
extJsonBytes, err := json.Marshal(ext)
if err != nil {
log.Println(err)
source.SetOperator(int32(intimate.OperatorError))
source.SetErrorMsg(sql.NullString{String: err.Error(), Valid: true})
continue
}
source.SetOperator(int32(intimate.OperatorOK))
source.SetExt(string(extJsonBytes))
store.Update(source)
}
} else {
log.Println("array error:", result.Str)
source.SetOperator(int32(intimate.OperatorError))
source.SetErrorMsg(sql.NullString{String: err.Error(), Valid: true})
continue
}
ext := make(map[string]interface{})
ext["user"] = string(resp.Content())
wf = cxt.Session().Get(userUrl + "/supporters")
resp, err = wf.Execute()
if err != nil {
log.Println(err)
source.SetOperator(int32(intimate.OperatorError))
source.SetErrorMsg(sql.NullString{String: err.Error(), Valid: true})
continue
}
ext["user_supporters"] = string(resp.Content())
wf = cxt.Session().Get("https://www.openrec.tv/live/" + userid)
resp, err = wf.Execute()
if err != nil {
log.Println(err)
source.SetOperator(int32(intimate.OperatorError))
source.SetErrorMsg(sql.NullString{String: err.Error(), Valid: true})
continue
}
ext["user_live"] = string(resp.Content())
extJsonBytes, err := json.Marshal(ext)
if err != nil {
log.Println(err)
source.SetOperator(int32(intimate.OperatorError))
source.SetErrorMsg(sql.NullString{String: err.Error(), Valid: true})
continue
}
source.SetOperator(int32(intimate.OperatorOK))
source.SetExt(string(extJsonBytes))
store.Update(source)
}
}

0
testfile/openrec_user.json Normal file
View File