test/intimate
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

1.重构命名, 简化代码, 加入build.sh

Browse Source 
2.重构数据字段结构, 使数据合理性提高.
3.测试通过openrec获取数据完整性.
TODO: 测试时区问题.
This commit is contained in:
eson
2020-07-17 18:21:38 +08:00
parent bb0f3845b7
commit e1040e69cd
15 changed files with 254 additions and 594 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
tasks/openrec/openrec_task1/source_openrec.go
View File

@@ -1,19 +0,0 @@
package main
/*
CREATE TABLE `source_openrec` (
uid bigint AUTO_INCREMENT,
`url` text NOT NULL,
`target_type` varchar(64) NOT NULL,
`source` longtext DEFAULT NULL,
`ext` json DEFAULT NULL,
`update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
`operator` int DEFAULT 0,
`error_msg` text DEFAULT NULL,
PRIMARY KEY(`uid`),
KEY `operator_idx` (`operator`),
KEY `update_time_idx` (`update_time`),
KEY `target_type_idx` (`target_type`)
);
*/

42
tasks/openrec/openrec_task1/task_openrec.go
View File

@@ -63,11 +63,19 @@ func (or *OpenrecRanking) Execute(cxt *hunter.TaskContext) {
break
}
wf := cxt.Temporary()
tp := cxt.Temporary()
content := resp.Content()
if len(content) <= 200 {
return
if len(content) <= 200 { // 末页退出
finishpoint := time.Now()
log.Println("任务结束休眠, 下次启动时间:", finishpoint.Add(time.Minute*30))
for time.Now().Sub(finishpoint) < time.Minute*30 {
time.Sleep(time.Second)
if atomic.LoadInt32(&loop) > 0 {
return
}
}
continue
}
result := gjson.ParseBytes(content)
@@ -76,39 +84,23 @@ func (or *OpenrecRanking) Execute(cxt *hunter.TaskContext) {
data := &intimate.Source{}
userid := User.Get("channel.id").String()
data.SetSource(sql.NullString{String: userid, Valid: len(userid) > 0})
data.SetUrl(wf.GetRawURL())
data.SetTargetType(string(intimate.TTOpenrecUser))
data.Source = sql.NullString{String: userid, Valid: len(userid) > 0}
data.Url = tp.GetRawURL()
data.TargetType = string(intimate.TTOpenrecUser)
store.Insert(data)
}
}
querys := wf.GetQuery()
querys := tp.GetQuery()
page, err := strconv.Atoi(querys.Get("page"))
if err != nil {
log.Println(err)
return
}
return
page++
querys.Set("page", strconv.Itoa(page))
wf.SetQuery(querys)
tp.SetQuery(querys)
time.Sleep(time.Second * 2)
}
}
// OpenrecUser 获取用户信息
type OpenrecUser struct {
hunter.PreCurlUrl
}
// Execute 执行方法
func (or *OpenrecUser) Execute(cxt *hunter.TaskContext) {
resp, err := cxt.Hunt()
if err != nil {
log.Println(err)
return
}
resp.Content()
}

0
tasks/openrec/openrec_task1/source_openrec_test.go → tasks/openrec/openrec_task1/task_openrec_test.go
View File

3
tasks/openrec/openrec_task2/.gitignore vendored
View File

@@ -1 +1,2 @@
openrec_task2
openrec_task2
log

31
tasks/openrec/openrec_task2/task_openrec.go
View File

@@ -44,24 +44,27 @@ func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) {
atomic.StoreInt32(&loop, 0)
}()
var lasterr error = nil
for atomic.LoadInt32(&loop) > 0 {
source, err := store.Pop(string(intimate.TTOpenrecUser))
if source == nil || err != nil {
log.Println(err)
if err != lasterr {
log.Println(err, lasterr)
lasterr = err
}
time.Sleep(time.Second * 2)
continue
}
userSource := &intimate.Source{}
userid := source.GetSource().String
userUrl := "https://www.openrec.tv/user/" + userid
userSource.SetUrl(userUrl)
userId := source.Source.String
userUrl := "https://www.openrec.tv/user/" + userId
wf := cxt.Session().Get(userUrl)
resp, err := wf.Execute()
source.SetUpdateTime(sql.NullTime{Time: time.Now(), Valid: true})
tp := cxt.Session().Get(userUrl)
resp, err := tp.Execute()
source.UpdateTime = sql.NullTime{Time: time.Now(), Valid: true}
if err != nil {
log.Println(err)
@@ -69,7 +72,7 @@ func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) {
continue
}
cookies := cxt.Session().GetCookies(wf.GetParsedURL())
cookies := cxt.Session().GetCookies(tp.GetParsedURL())
scurl := "https://www.openrec.tv/viewapp/api/v6/supporters?identify_id=sumomo_xqx&month=&Uuid=B96EE988-E3A2-4A44-A543-611A8B4BC683&Token=46598c320408bd69ae3c63298f6f4a3a97354175&Random=AZVXNAAXQVMOSVWNDPIQ&page_number=1 -H 'accept: application/json, text/javascript, */*; q=0.01' -H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36' -H 'cookie: uuid=B96EE988-E3A2-4A44-A543-611A8B4BC683;' --compressed"
curl := gcurl.ParseRawCURL(scurl)
@@ -99,7 +102,7 @@ func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) {
}
}
supportersQuery.Set("identify_id", source.GetSource().String)
supportersQuery.Set("identify_id", userId)
temporary.SetQuery(supportersQuery)
resp, err := temporary.Execute()
@@ -131,8 +134,8 @@ func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) {
ext["supporters"] = supporters
ext["user"] = string(resp.Content())
wf = cxt.Session().Get("https://www.openrec.tv/live/" + userid)
resp, err = wf.Execute()
tp = cxt.Session().Get("https://www.openrec.tv/live/" + userId)
resp, err = tp.Execute()
if err != nil {
log.Println(err)
store.UpdateError(source, err)
@@ -147,8 +150,8 @@ func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) {
continue
}
source.SetOperator(int32(intimate.OperatorOK))
source.SetExt(string(extJsonBytes))
source.Operator = int32(intimate.OperatorOK)
source.Ext = string(extJsonBytes)
store.Update(source)
}