From 5cbb17d6d43c68d05ed699d5a358f7490ca12fd7 Mon Sep 17 00:00:00 2001 From: eson Date: Wed, 8 Jul 2020 15:02:55 +0800 Subject: [PATCH] =?UTF-8?q?add:=20Pop=20=E9=98=9F=E5=88=97.=20=E4=BB=A5mys?= =?UTF-8?q?ql=20=E8=A1=A8=E7=9A=84=E6=A0=87=E5=BF=97=E5=81=9A=E4=B8=80?= =?UTF-8?q?=E4=B8=AA=E9=98=9F=E5=88=97.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go.mod | 1 + go.sum | 14 +++ source.go | 98 +++++++++++++++++ sql/intimate_extractor.sql | 1 - store.go | 89 ++++++++++++++-- store_test.go | 5 + tasks/openrec/source_openrec.go | 179 -------------------------------- tasks/openrec/task_openrec.go | 104 +++++++++++++++++++ 8 files changed, 303 insertions(+), 188 deletions(-) create mode 100644 source.go create mode 100644 tasks/openrec/task_openrec.go diff --git a/go.mod b/go.mod index c38d4d3..00a84d7 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/satori/go.uuid v1.2.0 github.com/tidwall/gjson v1.6.0 github.com/tidwall/pretty v1.0.1 // indirect + golang.org/x/tools v0.0.0-20200708003708-134513de8882 // indirect gopkg.in/yaml.v2 v2.2.2 honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a ) diff --git a/go.sum b/go.sum index 9a57a03..942a89d 100644 --- a/go.sum +++ b/go.sum @@ -71,10 +71,13 @@ github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/pretty v1.0.1 h1:WE4RBSZ1x6McVVC8S/Md+Qse8YUv6HRObAx6ke00NY8= github.com/tidwall/pretty v1.0.1/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= @@ -84,6 +87,7 @@ golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTk golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -96,6 +100,8 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344 h1:vGXIOMxbNfDTk/aXCmfdLgkrSV+Z2tcbze+pEc3v5W4= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -104,6 +110,7 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -127,7 +134,14 @@ golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190624190245-7f2218787638 h1:uIfBkD8gLczr4XDgYpt/qJYds2YJwZRNw4zs7wSnNhk= golang.org/x/tools v0.0.0-20190624190245-7f2218787638/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200708003708-134513de8882 h1:x4Two2lSwHxTqR+eal4lB4ydUnTvmDDpPQeL92ZHDgA= +golang.org/x/tools v0.0.0-20200708003708-134513de8882/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= diff --git a/source.go b/source.go new file mode 100644 index 0000000..0452ffe --- /dev/null +++ b/source.go @@ -0,0 +1,98 @@ +package intimate + +import ( + "database/sql" + "time" +) + +// Source 的结构体 +type Source struct { + Uid sql.NullInt64 // + Url string // + TargetType string // + Source sql.NullString // + Ext interface{} // + UpdateTime time.Time // + Operator int32 // + ErrorMsg sql.NullString // +} + +// GetErrorMsg Get return ErrorMsg sql.NullString +func (so *Source) GetErrorMsg() sql.NullString { + return so.ErrorMsg +} + +// SetErrorMsg Set ErrorMsg sql.NullString +func (so *Source) SetErrorMsg(ErrorMsg sql.NullString) { + so.ErrorMsg = ErrorMsg +} + +// GetOperator Get return Operator sql.NullInt32 +func (so *Source) GetOperator() int32 { + return so.Operator +} + +// SetOperator Set Operator sql.NullInt32 +func (so *Source) SetOperator(Operator int32) { + so.Operator = Operator +} + +// GetUpdateTime Get return UpdateTime time.Time +func (so *Source) GetUpdateTime() time.Time { + return so.UpdateTime +} + +// SetUpdateTime Set UpdateTime time.Time +func (so *Source) SetUpdateTime(UpdateTime time.Time) { + so.UpdateTime = UpdateTime +} + +// GetExt Get return Ext interface{} +func (so *Source) GetExt() interface{} { + return so.Ext +} + +// SetExt Set Ext interface{} +func (so *Source) SetExt(Ext interface{}) { + so.Ext = Ext +} + +// GetSource Get return Source sql.NullString +func (so *Source) GetSource() sql.NullString { + return so.Source +} + +// SetSource Set Source sql.NullString +func (so *Source) SetSource(Source sql.NullString) { + so.Source = Source +} + +// GetTargetType Get return TargetType string +func (so *Source) GetTargetType() string { + return so.TargetType +} + +// SetTargetType Set TargetType string +func (so *Source) SetTargetType(TargetType string) { + so.TargetType = TargetType +} + +// GetUrl Get return Url string +func (so *Source) GetUrl() string { + return so.Url +} + +// SetUrl Set Url string +func (so *Source) SetUrl(Url string) { + so.Url = Url +} + +// GetUid Get return Uid sql.NullInt64 +func (so *Source) GetUid() sql.NullInt64 { + return so.Uid +} + +// SetUid Set Uid sql.NullInt64 +func (so *Source) SetUid(Uid sql.NullInt64) { + so.Uid = Uid +} diff --git a/sql/intimate_extractor.sql b/sql/intimate_extractor.sql index c1d12e9..a114820 100644 --- a/sql/intimate_extractor.sql +++ b/sql/intimate_extractor.sql @@ -17,7 +17,6 @@ CREATE TABLE IF NOT EXISTS `anchor_info` ( KEY `channel_idx` (`channel`), KEY `show_type_idx` (`show_type`), KEY `update_time_idx` (`update_time`) - ); CREATE TABLE IF NOT EXISTS `show_log` ( diff --git a/store.go b/store.go index 90a0323..93a9d36 100644 --- a/store.go +++ b/store.go @@ -2,14 +2,15 @@ package intimate import ( "database/sql" + "errors" "log" "time" _ "github.com/go-sql-driver/mysql" ) -// ISource 源接口结构 -type ISource interface { +// IGetSource 源接口结构 +type IGetSource interface { GetUrl() string // GetTargetType() string // GetSource() sql.NullString // @@ -17,27 +18,99 @@ type ISource interface { GetUpdateTime() time.Time // GetOperator() int32 // GetErrorMsg() sql.NullString // - } +type IUpdateSource interface { + IGetSource + + SetExt(ext interface{}) // + SetUpdateTime(ut time.Time) // + SetOperator(operator int32) // + SetErrorMsg(emsg sql.NullString) // +} + +// OperatorFlag 标志 +type OperatorFlag int32 + +const ( + // OperatorWait 等待被处理 + OperatorWait OperatorFlag = 1000 + // OperatorError 错误标志 + OperatorError OperatorFlag = 10000 +) + // Store 储存 type Store struct { - db *sql.DB + table string + db *sql.DB } // NewStore 创建一个存储实例 -func NewStore() *Store { +func NewStore(table string) *Store { db, err := sql.Open("mysql", InitConfig.Database.URI) if err != nil { panic(err) } - return &Store{db: db} + return &Store{table: table, db: db} } // Save 储存数据 -func (store *Store) Save(isource ISource) { - _, err := store.db.Exec("insert into `platform_openrec`(url, target_type, source, ext, operator, error_msg) values(?,?,?,?,?,?)", isource.GetUrl(), isource.GetTargetType(), isource.GetSource(), isource.GetExt(), isource.GetOperator(), isource.GetErrorMsg()) +func (store *Store) Save(isource IGetSource) { + _, err := store.db.Exec("insert into `source_openrec`(url, target_type, source, ext, operator, error_msg) values(?,?,?,?,?,?)", isource.GetUrl(), isource.GetTargetType(), isource.GetSource(), isource.GetExt(), isource.GetOperator(), isource.GetErrorMsg()) if err != nil { log.Fatalln(err) } } + +// Pop 储存数据 +func (store *Store) Pop(targetType string, operators ...int32) (IUpdateSource, error) { + + tx, err := store.db.Begin() + if err != nil { + log.Println(err, targetType) + return nil, err + } + var args = []interface{}{targetType} + selectSQL := `select uid, url, target_type, source, ext, operator from ` + store.table + ` where target_type = ?` + if len(operators) == 0 { + selectSQL += " and operator = ?" + args = append(args, 0) + } else { + for _, operator := range operators { + selectSQL += " and operator = ?" + args = append(args, operator) + } + } + + log.Println(selectSQL + ` limit 1 for update`) + row := tx.QueryRow(selectSQL, args...) + + defer func() { + err := tx.Commit() + if err != nil { + log.Println(err) + err = tx.Rollback() + if err != nil { + log.Println(err) + } + } + }() + + if row != nil { + s := &Source{} + // uid, url, target_type, source, ext, operator + err = row.Scan(&s.Uid, &s.Url, &s.TargetType, &s.Source, &s.Ext, &s.Operator) + if err != nil { + log.Println(err, targetType) + _, err = tx.Exec("update "+store.table+" set error_msg = ?, operator = ? where uid = ?", OperatorError, s.Uid) + if err != nil { + log.Println(err) + } + return nil, err + } + _, err = tx.Exec("update "+store.table+" set operator = ? where uid = ?", OperatorWait, s.Uid) + return s, nil + } + + return nil, errors.New("TaskQueue is nil") +} diff --git a/store_test.go b/store_test.go index 7889937..c19bcd7 100644 --- a/store_test.go +++ b/store_test.go @@ -22,3 +22,8 @@ func TestStoreInsertCase1(t *testing.T) { // t.Error(err, len(uuid.NewV4().String())) } + +func TestStorePopCase1(t *testing.T) { + store := NewStore("source_openrec") + t.Error(store.Pop("openrec_ranking")) +} diff --git a/tasks/openrec/source_openrec.go b/tasks/openrec/source_openrec.go index 897db8b..ca805f1 100644 --- a/tasks/openrec/source_openrec.go +++ b/tasks/openrec/source_openrec.go @@ -1,40 +1,5 @@ package main -import ( - "database/sql" - "log" - "strconv" - "time" - - "intimate" - - "github.com/474420502/hunter" -) - -var targetTypeRanking = "openrec_ranking" -var openrecRanking *OpenrecRanking - -// store 源存储实例, 为存储源数据的实现. 表格具体参考sql/intimate_source.sql -var store *intimate.Store = intimate.NewStore() - -func init() { - - openrecRanking = &OpenrecRanking{} - openrecRanking.PreCurlUrl = `curl 'https://public.openrec.tv/external/api/v5/channel-ranks?period=monthly&date=&tag=&page=1' \ - -H 'authority: public.openrec.tv' \ - -H 'accept: application/json, text/javascript, */*; q=0.01' \ - -H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36' \ - -H 'origin: https://www.openrec.tv' \ - -H 'sec-fetch-site: same-site' \ - -H 'sec-fetch-mode: cors' \ - -H 'sec-fetch-dest: empty' \ - -H 'referer: https://www.openrec.tv/ranking' \ - -H 'accept-language: zh-CN,zh;q=0.9' \ - -H 'if-none-match: W/"25edb-aUYBdmLqZcr6DW4ZWKX9r2aqolg"' \ - --compressed` - -} - /* CREATE TABLE `source_openrec` ( uid bigint AUTO_INCREMENT, @@ -52,147 +17,3 @@ CREATE TABLE `source_openrec` ( KEY `target_type_idx` (`target_type`) ); */ - -type SourceOpenrec struct { - Uid sql.NullInt64 // - Url string // - TargetType string // - Source sql.NullString // - Ext interface{} // - UpdateTime time.Time // - Operator int32 // - ErrorMsg sql.NullString // -} - -// GetErrorMsg Get return ErrorMsg sql.NullString -func (so *SourceOpenrec) GetErrorMsg() sql.NullString { - return so.ErrorMsg -} - -// SetErrorMsg Set ErrorMsg sql.NullString -func (so *SourceOpenrec) SetErrorMsg(ErrorMsg sql.NullString) { - so.ErrorMsg = ErrorMsg -} - -// GetOperator Get return Operator sql.NullInt32 -func (so *SourceOpenrec) GetOperator() int32 { - return so.Operator -} - -// SetOperator Set Operator sql.NullInt32 -func (so *SourceOpenrec) SetOperator(Operator int32) { - so.Operator = Operator -} - -// GetUpdateTime Get return UpdateTime time.Time -func (so *SourceOpenrec) GetUpdateTime() time.Time { - return so.UpdateTime -} - -// SetUpdateTime Set UpdateTime time.Time -func (so *SourceOpenrec) SetUpdateTime(UpdateTime time.Time) { - so.UpdateTime = UpdateTime -} - -// GetExt Get return Ext interface{} -func (so *SourceOpenrec) GetExt() interface{} { - return so.Ext -} - -// SetExt Set Ext interface{} -func (so *SourceOpenrec) SetExt(Ext interface{}) { - so.Ext = Ext -} - -// GetSource Get return Source sql.NullString -func (so *SourceOpenrec) GetSource() sql.NullString { - return so.Source -} - -// SetSource Set Source sql.NullString -func (so *SourceOpenrec) SetSource(Source sql.NullString) { - so.Source = Source -} - -// GetTargetType Get return TargetType string -func (so *SourceOpenrec) GetTargetType() string { - return so.TargetType -} - -// SetTargetType Set TargetType string -func (so *SourceOpenrec) SetTargetType(TargetType string) { - so.TargetType = TargetType -} - -// GetUrl Get return Url string -func (so *SourceOpenrec) GetUrl() string { - return so.Url -} - -// SetUrl Set Url string -func (so *SourceOpenrec) SetUrl(Url string) { - so.Url = Url -} - -// GetUid Get return Uid sql.NullInt64 -func (so *SourceOpenrec) GetUid() sql.NullInt64 { - return so.Uid -} - -// SetUid Set Uid sql.NullInt64 -func (so *SourceOpenrec) SetUid(Uid sql.NullInt64) { - so.Uid = Uid -} - -// OpenrecRanking 获取排名任务 -type OpenrecRanking struct { - hunter.PreCurlUrl -} - -// Execute 执行方法 -func (or *OpenrecRanking) Execute(cxt *hunter.TaskContext) { - - for { - - errorMsg := sql.NullString{Valid: false} - resp, err := cxt.Hunt() - if err != nil { - log.Println(err) - break - } - - wf := cxt.Workflow() - - data := &SourceOpenrec{} - content := resp.Content() - if len(content) <= 200 { - return - } - - data.SetSource(sql.NullString{String: string(content), Valid: len(content) > 0}) - data.SetUrl(wf.GetRawURL()) - data.SetTargetType(targetTypeRanking) - - querys := wf.GetQuery() - page, err := strconv.Atoi(querys.Get("page")) - if err != nil { - log.Println(err) - errorMsg.String = err.Error() - errorMsg.Valid = true - - data.SetErrorMsg(errorMsg) - data.SetOperator(10000) - store.Save(data) - return - } - - page++ - querys.Set("page", strconv.Itoa(page)) - wf.SetQuery(querys) - - data.SetErrorMsg(errorMsg) - store.Save(data) - - time.Sleep(time.Second * 2) - } -} diff --git a/tasks/openrec/task_openrec.go b/tasks/openrec/task_openrec.go new file mode 100644 index 0000000..290268a --- /dev/null +++ b/tasks/openrec/task_openrec.go @@ -0,0 +1,104 @@ +package main + +import ( + "database/sql" + "intimate" + "log" + "strconv" + "time" + + "github.com/474420502/hunter" +) + +var targetTypeRanking = "openrec_ranking" +var openrecRanking *OpenrecRanking + +// store 源存储实例, 为存储源数据的实现. 表格具体参考sql/intimate_source.sql +var store *intimate.Store = intimate.NewStore("source_openrec") + +func init() { + + openrecRanking = &OpenrecRanking{} + openrecRanking.PreCurlUrl = `curl 'https://public.openrec.tv/external/api/v5/channel-ranks?period=monthly&date=&tag=&page=1' \ + -H 'authority: public.openrec.tv' \ + -H 'accept: application/json, text/javascript, */*; q=0.01' \ + -H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36' \ + -H 'origin: https://www.openrec.tv' \ + -H 'sec-fetch-site: same-site' \ + -H 'sec-fetch-mode: cors' \ + -H 'sec-fetch-dest: empty' \ + -H 'referer: https://www.openrec.tv/ranking' \ + -H 'accept-language: zh-CN,zh;q=0.9' \ + -H 'if-none-match: W/"25edb-aUYBdmLqZcr6DW4ZWKX9r2aqolg"' \ + --compressed` + +} + +// OpenrecRanking 获取排名任务 +type OpenrecRanking struct { + hunter.PreCurlUrl +} + +// Execute 执行方法 +func (or *OpenrecRanking) Execute(cxt *hunter.TaskContext) { + + for { + + errorMsg := sql.NullString{Valid: false} + resp, err := cxt.Hunt() + if err != nil { + log.Println(err) + break + } + + wf := cxt.Workflow() + + data := &intimate.Source{} + content := resp.Content() + if len(content) <= 200 { + return + } + + data.SetSource(sql.NullString{String: string(content), Valid: len(content) > 0}) + data.SetUrl(wf.GetRawURL()) + data.SetTargetType(targetTypeRanking) + + querys := wf.GetQuery() + page, err := strconv.Atoi(querys.Get("page")) + if err != nil { + log.Println(err) + errorMsg.String = err.Error() + errorMsg.Valid = true + + data.SetErrorMsg(errorMsg) + data.SetOperator(10000) + store.Save(data) + return + } + + page++ + querys.Set("page", strconv.Itoa(page)) + wf.SetQuery(querys) + + data.SetErrorMsg(errorMsg) + store.Save(data) + + time.Sleep(time.Second * 2) + } +} + +// OpenrecUser 获取用户信息 +type OpenrecUser struct { + hunter.PreCurlUrl +} + +// Execute 执行方法 +func (or *OpenrecUser) Execute(cxt *hunter.TaskContext) { + resp, err := cxt.Hunt() + if err != nil { + log.Println(err) + return + } + + resp.Content() +}