add: Pop 队列. 以mysql 表的标志做一个队列.
This commit is contained in:
eson
@@ -1,40 +1,5 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"log"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"intimate"
|
||||
|
||||
"github.com/474420502/hunter"
|
||||
)
|
||||
|
||||
var targetTypeRanking = "openrec_ranking"
|
||||
var openrecRanking *OpenrecRanking
|
||||
|
||||
// store 源存储实例, 为存储源数据的实现. 表格具体参考sql/intimate_source.sql
|
||||
var store *intimate.Store = intimate.NewStore()
|
||||
|
||||
func init() {
|
||||
|
||||
openrecRanking = &OpenrecRanking{}
|
||||
openrecRanking.PreCurlUrl = `curl 'https://public.openrec.tv/external/api/v5/channel-ranks?period=monthly&date=&tag=&page=1' \
|
||||
-H 'authority: public.openrec.tv' \
|
||||
-H 'accept: application/json, text/javascript, */*; q=0.01' \
|
||||
-H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36' \
|
||||
-H 'origin: https://www.openrec.tv' \
|
||||
-H 'sec-fetch-site: same-site' \
|
||||
-H 'sec-fetch-mode: cors' \
|
||||
-H 'sec-fetch-dest: empty' \
|
||||
-H 'referer: https://www.openrec.tv/ranking' \
|
||||
-H 'accept-language: zh-CN,zh;q=0.9' \
|
||||
-H 'if-none-match: W/"25edb-aUYBdmLqZcr6DW4ZWKX9r2aqolg"' \
|
||||
--compressed`
|
||||
|
||||
}
|
||||
|
||||
/*
|
||||
CREATE TABLE `source_openrec` (
|
||||
uid bigint AUTO_INCREMENT,
|
||||
@@ -52,147 +17,3 @@ CREATE TABLE `source_openrec` (
|
||||
KEY `target_type_idx` (`target_type`)
|
||||
);
|
||||
*/
|
||||
|
||||
type SourceOpenrec struct {
|
||||
Uid sql.NullInt64 //
|
||||
Url string //
|
||||
TargetType string //
|
||||
Source sql.NullString //
|
||||
Ext interface{} //
|
||||
UpdateTime time.Time //
|
||||
Operator int32 //
|
||||
ErrorMsg sql.NullString //
|
||||
}
|
||||
|
||||
// GetErrorMsg Get return ErrorMsg sql.NullString
|
||||
func (so *SourceOpenrec) GetErrorMsg() sql.NullString {
|
||||
return so.ErrorMsg
|
||||
}
|
||||
|
||||
// SetErrorMsg Set ErrorMsg sql.NullString
|
||||
func (so *SourceOpenrec) SetErrorMsg(ErrorMsg sql.NullString) {
|
||||
so.ErrorMsg = ErrorMsg
|
||||
}
|
||||
|
||||
// GetOperator Get return Operator sql.NullInt32
|
||||
func (so *SourceOpenrec) GetOperator() int32 {
|
||||
return so.Operator
|
||||
}
|
||||
|
||||
// SetOperator Set Operator sql.NullInt32
|
||||
func (so *SourceOpenrec) SetOperator(Operator int32) {
|
||||
so.Operator = Operator
|
||||
}
|
||||
|
||||
// GetUpdateTime Get return UpdateTime time.Time
|
||||
func (so *SourceOpenrec) GetUpdateTime() time.Time {
|
||||
return so.UpdateTime
|
||||
}
|
||||
|
||||
// SetUpdateTime Set UpdateTime time.Time
|
||||
func (so *SourceOpenrec) SetUpdateTime(UpdateTime time.Time) {
|
||||
so.UpdateTime = UpdateTime
|
||||
}
|
||||
|
||||
// GetExt Get return Ext interface{}
|
||||
func (so *SourceOpenrec) GetExt() interface{} {
|
||||
return so.Ext
|
||||
}
|
||||
|
||||
// SetExt Set Ext interface{}
|
||||
func (so *SourceOpenrec) SetExt(Ext interface{}) {
|
||||
so.Ext = Ext
|
||||
}
|
||||
|
||||
// GetSource Get return Source sql.NullString
|
||||
func (so *SourceOpenrec) GetSource() sql.NullString {
|
||||
return so.Source
|
||||
}
|
||||
|
||||
// SetSource Set Source sql.NullString
|
||||
func (so *SourceOpenrec) SetSource(Source sql.NullString) {
|
||||
so.Source = Source
|
||||
}
|
||||
|
||||
// GetTargetType Get return TargetType string
|
||||
func (so *SourceOpenrec) GetTargetType() string {
|
||||
return so.TargetType
|
||||
}
|
||||
|
||||
// SetTargetType Set TargetType string
|
||||
func (so *SourceOpenrec) SetTargetType(TargetType string) {
|
||||
so.TargetType = TargetType
|
||||
}
|
||||
|
||||
// GetUrl Get return Url string
|
||||
func (so *SourceOpenrec) GetUrl() string {
|
||||
return so.Url
|
||||
}
|
||||
|
||||
// SetUrl Set Url string
|
||||
func (so *SourceOpenrec) SetUrl(Url string) {
|
||||
so.Url = Url
|
||||
}
|
||||
|
||||
// GetUid Get return Uid sql.NullInt64
|
||||
func (so *SourceOpenrec) GetUid() sql.NullInt64 {
|
||||
return so.Uid
|
||||
}
|
||||
|
||||
// SetUid Set Uid sql.NullInt64
|
||||
func (so *SourceOpenrec) SetUid(Uid sql.NullInt64) {
|
||||
so.Uid = Uid
|
||||
}
|
||||
|
||||
// OpenrecRanking 获取排名任务
|
||||
type OpenrecRanking struct {
|
||||
hunter.PreCurlUrl
|
||||
}
|
||||
|
||||
// Execute 执行方法
|
||||
func (or *OpenrecRanking) Execute(cxt *hunter.TaskContext) {
|
||||
|
||||
for {
|
||||
|
||||
errorMsg := sql.NullString{Valid: false}
|
||||
resp, err := cxt.Hunt()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
break
|
||||
}
|
||||
|
||||
wf := cxt.Workflow()
|
||||
|
||||
data := &SourceOpenrec{}
|
||||
content := resp.Content()
|
||||
if len(content) <= 200 {
|
||||
return
|
||||
}
|
||||
|
||||
data.SetSource(sql.NullString{String: string(content), Valid: len(content) > 0})
|
||||
data.SetUrl(wf.GetRawURL())
|
||||
data.SetTargetType(targetTypeRanking)
|
||||
|
||||
querys := wf.GetQuery()
|
||||
page, err := strconv.Atoi(querys.Get("page"))
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
errorMsg.String = err.Error()
|
||||
errorMsg.Valid = true
|
||||
|
||||
data.SetErrorMsg(errorMsg)
|
||||
data.SetOperator(10000)
|
||||
store.Save(data)
|
||||
return
|
||||
}
|
||||
|
||||
page++
|
||||
querys.Set("page", strconv.Itoa(page))
|
||||
wf.SetQuery(querys)
|
||||
|
||||
data.SetErrorMsg(errorMsg)
|
||||
store.Save(data)
|
||||
|
||||
time.Sleep(time.Second * 2)
|
||||
}
|
||||
}
|
||||
|
||||
104
tasks/openrec/task_openrec.go
Normal file
104
tasks/openrec/task_openrec.go
Normal file
@@ -0,0 +1,104 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"intimate"
|
||||
"log"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/474420502/hunter"
|
||||
)
|
||||
|
||||
var targetTypeRanking = "openrec_ranking"
|
||||
var openrecRanking *OpenrecRanking
|
||||
|
||||
// store 源存储实例, 为存储源数据的实现. 表格具体参考sql/intimate_source.sql
|
||||
var store *intimate.Store = intimate.NewStore("source_openrec")
|
||||
|
||||
func init() {
|
||||
|
||||
openrecRanking = &OpenrecRanking{}
|
||||
openrecRanking.PreCurlUrl = `curl 'https://public.openrec.tv/external/api/v5/channel-ranks?period=monthly&date=&tag=&page=1' \
|
||||
-H 'authority: public.openrec.tv' \
|
||||
-H 'accept: application/json, text/javascript, */*; q=0.01' \
|
||||
-H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36' \
|
||||
-H 'origin: https://www.openrec.tv' \
|
||||
-H 'sec-fetch-site: same-site' \
|
||||
-H 'sec-fetch-mode: cors' \
|
||||
-H 'sec-fetch-dest: empty' \
|
||||
-H 'referer: https://www.openrec.tv/ranking' \
|
||||
-H 'accept-language: zh-CN,zh;q=0.9' \
|
||||
-H 'if-none-match: W/"25edb-aUYBdmLqZcr6DW4ZWKX9r2aqolg"' \
|
||||
--compressed`
|
||||
|
||||
}
|
||||
|
||||
// OpenrecRanking 获取排名任务
|
||||
type OpenrecRanking struct {
|
||||
hunter.PreCurlUrl
|
||||
}
|
||||
|
||||
// Execute 执行方法
|
||||
func (or *OpenrecRanking) Execute(cxt *hunter.TaskContext) {
|
||||
|
||||
for {
|
||||
|
||||
errorMsg := sql.NullString{Valid: false}
|
||||
resp, err := cxt.Hunt()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
break
|
||||
}
|
||||
|
||||
wf := cxt.Workflow()
|
||||
|
||||
data := &intimate.Source{}
|
||||
content := resp.Content()
|
||||
if len(content) <= 200 {
|
||||
return
|
||||
}
|
||||
|
||||
data.SetSource(sql.NullString{String: string(content), Valid: len(content) > 0})
|
||||
data.SetUrl(wf.GetRawURL())
|
||||
data.SetTargetType(targetTypeRanking)
|
||||
|
||||
querys := wf.GetQuery()
|
||||
page, err := strconv.Atoi(querys.Get("page"))
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
errorMsg.String = err.Error()
|
||||
errorMsg.Valid = true
|
||||
|
||||
data.SetErrorMsg(errorMsg)
|
||||
data.SetOperator(10000)
|
||||
store.Save(data)
|
||||
return
|
||||
}
|
||||
|
||||
page++
|
||||
querys.Set("page", strconv.Itoa(page))
|
||||
wf.SetQuery(querys)
|
||||
|
||||
data.SetErrorMsg(errorMsg)
|
||||
store.Save(data)
|
||||
|
||||
time.Sleep(time.Second * 2)
|
||||
}
|
||||
}
|
||||
|
||||
// OpenrecUser 获取用户信息
|
||||
type OpenrecUser struct {
|
||||
hunter.PreCurlUrl
|
||||
}
|
||||
|
||||
// Execute 执行方法
|
||||
func (or *OpenrecUser) Execute(cxt *hunter.TaskContext) {
|
||||
resp, err := cxt.Hunt()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
return
|
||||
}
|
||||
|
||||
resp.Content()
|
||||
}
|
||||
Reference in New Issue
Block a user