intimate/source_openrec.go

package main

import (
	"database/sql"
	"log"
	"strconv"
	"time"

	"github.com/474420502/hunter"
)

var targetTypeRanking = "openrec_ranking"
var openrecRanking *OpenrecRanking

// storeOpenrec 源存储实例, 为存储源数据的实现. 表格具体参考sql/intimate_source.sql
var storeOpenrec *Store

func init() {

	openrecRanking = &OpenrecRanking{}
	openrecRanking.PreCurlUrl = `curl 'https://public.openrec.tv/external/api/v5/channel-ranks?period=monthly&date=&tag=&page=1' \
	-H 'authority: public.openrec.tv' \
	-H 'accept: application/json, text/javascript, */*; q=0.01' \
	-H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36' \
	-H 'origin: https://www.openrec.tv' \
	-H 'sec-fetch-site: same-site' \
	-H 'sec-fetch-mode: cors' \
	-H 'sec-fetch-dest: empty' \
	-H 'referer: https://www.openrec.tv/ranking' \
	-H 'accept-language: zh-CN,zh;q=0.9' \
	-H 'if-none-match: W/"25edb-aUYBdmLqZcr6DW4ZWKX9r2aqolg"' \
	--compressed`
}

/*
CREATE TABLE  `source_openrec` (
	uid bigint AUTO_INCREMENT,
	`url` text NOT NULL,
	`target_type` varchar(64) NOT NULL,
	`source` longtext DEFAULT NULL,
	`ext` json DEFAULT NULL,

	`update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
	`operator` int DEFAULT 0,
	`error_msg` text DEFAULT NULL,
	PRIMARY KEY(`uid`),
	KEY `operator_idx` (`operator`),
	KEY `update_time_idx` (`update_time`),
	KEY `target_type_idx` (`target_type`)
 );
*/

type SourceOpenrec struct {
	Uid        sql.NullInt64  //
	Url        string         //
	TargetType string         //
	Source     sql.NullString //
	Ext        interface{}    //
	UpdateTime time.Time      //
	Operator   int32          //
	ErrorMsg   sql.NullString //
}

// GetErrorMsg Get return ErrorMsg sql.NullString
func (so *SourceOpenrec) GetErrorMsg() sql.NullString {
	return so.ErrorMsg
}

// SetErrorMsg Set ErrorMsg sql.NullString
func (so *SourceOpenrec) SetErrorMsg(ErrorMsg sql.NullString) {
	so.ErrorMsg = ErrorMsg
}

// GetOperator Get return Operator sql.NullInt32
func (so *SourceOpenrec) GetOperator() int32 {
	return so.Operator
}

// SetOperator Set Operator sql.NullInt32
func (so *SourceOpenrec) SetOperator(Operator int32) {
	so.Operator = Operator
}

// GetUpdateTime Get return UpdateTime time.Time
func (so *SourceOpenrec) GetUpdateTime() time.Time {
	return so.UpdateTime
}

// SetUpdateTime Set UpdateTime time.Time
func (so *SourceOpenrec) SetUpdateTime(UpdateTime time.Time) {
	so.UpdateTime = UpdateTime
}

// GetExt Get return Ext interface{}
func (so *SourceOpenrec) GetExt() interface{} {
	return so.Ext
}

// SetExt Set Ext interface{}
func (so *SourceOpenrec) SetExt(Ext interface{}) {
	so.Ext = Ext
}

// GetSource Get return Source sql.NullString
func (so *SourceOpenrec) GetSource() sql.NullString {
	return so.Source
}

// SetSource Set Source sql.NullString
func (so *SourceOpenrec) SetSource(Source sql.NullString) {
	so.Source = Source
}

// GetTargetType Get return TargetType string
func (so *SourceOpenrec) GetTargetType() string {
	return so.TargetType
}

// SetTargetType Set TargetType string
func (so *SourceOpenrec) SetTargetType(TargetType string) {
	so.TargetType = TargetType
}

// GetUrl Get return Url string
func (so *SourceOpenrec) GetUrl() string {
	return so.Url
}

// SetUrl Set Url string
func (so *SourceOpenrec) SetUrl(Url string) {
	so.Url = Url
}

// GetUid Get return Uid sql.NullInt64
func (so *SourceOpenrec) GetUid() sql.NullInt64 {
	return so.Uid
}

// SetUid Set Uid sql.NullInt64
func (so *SourceOpenrec) SetUid(Uid sql.NullInt64) {
	so.Uid = Uid
}

// OpenrecRanking 获取排名任务
type OpenrecRanking struct {
	hunter.PreCurlUrl
}

// Execute 执行方法
func (or *OpenrecRanking) Execute(cxt *hunter.TaskContext) {

	for {

		errorMsg := sql.NullString{Valid: false}
		resp, err := cxt.Hunt()
		if err != nil {
			log.Println(err)
			break
		}

		wf := cxt.Workflow()

		data := &SourceOpenrec{}
		content := resp.Content()
		log.Println(len(content))
		if len(content) <= 300 {
			return
		}

		data.SetSource(sql.NullString{String: string(content), Valid: len(content) > 0})
		data.SetUrl(wf.GetRawURL())
		data.SetTargetType(targetTypeRanking)

		querys := wf.GetQuery()
		page, err := strconv.Atoi(querys.Get("page"))
		if err != nil {
			log.Println(err)
			errorMsg.String = err.Error()
			errorMsg.Valid = true

			data.SetErrorMsg(errorMsg)
			data.SetOperator(10000)
			storeOpenrec.Save(data)
			return
		}

		page++
		querys.Set("page", strconv.Itoa(page))
		wf.SetQuery(querys)

		data.SetErrorMsg(errorMsg)
		storeOpenrec.Save(data)

		time.Sleep(time.Second * 2)
	}
}