TODO: openrec user data. isource update interface

tasks/openrec/openrec_task1/.gitignore

@@ -0,0 +1 @@
+task1

tasks/openrec/openrec_task1/config.yaml

@@ -0,0 +1 @@
+../../../config.yaml

tasks/openrec/openrec_task1/main.go

@@ -0,0 +1,8 @@
+package main
+
+import "github.com/474420502/hunter"
+
+func main() {
+	ht := hunter.NewHunter(openrecRanking)
+	ht.Execute()
+}

tasks/openrec/openrec_task1/source_openrec.go

@@ -0,0 +1,19 @@
+package main
+
+/*
+CREATE TABLE  `source_openrec` (
+	uid bigint AUTO_INCREMENT,
+	`url` text NOT NULL,
+	`target_type` varchar(64) NOT NULL,
+	`source` longtext DEFAULT NULL,
+	`ext` json DEFAULT NULL,
+
+	`update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+	`operator` int DEFAULT 0,
+	`error_msg` text DEFAULT NULL,
+	PRIMARY KEY(`uid`),
+	KEY `operator_idx` (`operator`),
+	KEY `update_time_idx` (`update_time`),
+	KEY `target_type_idx` (`target_type`)
+ );
+*/

tasks/openrec/openrec_task1/source_openrec_test.go

@@ -0,0 +1,62 @@
+package main
+
+import (
+	"testing"
+
+	"github.com/tidwall/gjson"
+
+	"github.com/474420502/hunter"
+)
+
+// OpenrecRanking 获取排名任务
+type OpenrecRankingTest struct {
+	hunter.PreCurlUrl
+}
+
+// Execute 执行方法
+func (or *OpenrecRankingTest) Execute(cxt *hunter.TaskContext) {
+
+	resp, err := cxt.Hunt()
+	if err != nil {
+		panic(err)
+	}
+	t := cxt.GetShare("test").(*testing.T)
+	if !gjson.ValidBytes(resp.Content()) {
+		t.Error("source is not json format.")
+	}
+	result := gjson.GetBytes(resp.Content(), "0.rank")
+	if result.Int() != 1 {
+		t.Error("rank is error. result raw is ", result.Raw)
+	}
+
+	if cxt.Workflow().GetQuery().Get("page") != "1" {
+		t.Error("workflow page error")
+	}
+	// t.Error(string(resp.Content()))
+}
+
+func TestRanking(t *testing.T) {
+	curlBash := `curl 'https://public.openrec.tv/external/api/v5/channel-ranks?period=monthly&date=&tag=&page=1' \
+	-H 'authority: public.openrec.tv' \
+	-H 'accept: application/json, text/javascript, */*; q=0.01' \
+	-H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36' \
+	-H 'origin: https://www.openrec.tv' \
+	-H 'sec-fetch-site: same-site' \
+	-H 'sec-fetch-mode: cors' \
+	-H 'sec-fetch-dest: empty' \
+	-H 'referer: https://www.openrec.tv/ranking' \
+	-H 'accept-language: zh-CN,zh;q=0.9' \
+	-H 'if-none-match: W/"25edb-aUYBdmLqZcr6DW4ZWKX9r2aqolg"' \
+	--compressed`
+
+	ort := &OpenrecRankingTest{}
+	ort.PreCurlUrl = hunter.PreCurlUrl(curlBash)
+	ht := hunter.NewHunter(ort)
+	ht.SetShare("test", t)
+	ht.Execute()
+}
+
+func TestRankingInsert(t *testing.T) {
+	ht := hunter.NewHunter(openrecRanking)
+	ht.Execute()
+}

tasks/openrec/openrec_task1/task_openrec.go

@@ -0,0 +1,104 @@
+package main
+
+import (
+	"database/sql"
+	"intimate"
+	"log"
+	"strconv"
+	"time"
+
+	"github.com/474420502/hunter"
+)
+
+var targetTypeRanking = "openrec_ranking"
+var openrecRanking *OpenrecRanking
+
+// store  源存储实例, 为存储源数据的实现. 表格具体参考sql/intimate_source.sql
+var store *intimate.Store = intimate.NewStore("source_openrec")
+
+func init() {
+
+	openrecRanking = &OpenrecRanking{}
+	openrecRanking.PreCurlUrl = `curl 'https://public.openrec.tv/external/api/v5/channel-ranks?period=monthly&date=&tag=&page=1' \
+	-H 'authority: public.openrec.tv' \
+	-H 'accept: application/json, text/javascript, */*; q=0.01' \
+	-H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36' \
+	-H 'origin: https://www.openrec.tv' \
+	-H 'sec-fetch-site: same-site' \
+	-H 'sec-fetch-mode: cors' \
+	-H 'sec-fetch-dest: empty' \
+	-H 'referer: https://www.openrec.tv/ranking' \
+	-H 'accept-language: zh-CN,zh;q=0.9' \
+	-H 'if-none-match: W/"25edb-aUYBdmLqZcr6DW4ZWKX9r2aqolg"' \
+	--compressed`
+
+}
+
+// OpenrecRanking 获取排名任务
+type OpenrecRanking struct {
+	hunter.PreCurlUrl
+}
+
+// Execute 执行方法
+func (or *OpenrecRanking) Execute(cxt *hunter.TaskContext) {
+
+	for {
+
+		errorMsg := sql.NullString{Valid: false}
+		resp, err := cxt.Hunt()
+		if err != nil {
+			log.Println(err)
+			break
+		}
+
+		wf := cxt.Workflow()
+
+		data := &intimate.Source{}
+		content := resp.Content()
+		if len(content) <= 200 {
+			return
+		}
+
+		data.SetSource(sql.NullString{String: string(content), Valid: len(content) > 0})
+		data.SetUrl(wf.GetRawURL())
+		data.SetTargetType(targetTypeRanking)
+
+		querys := wf.GetQuery()
+		page, err := strconv.Atoi(querys.Get("page"))
+		if err != nil {
+			log.Println(err)
+			errorMsg.String = err.Error()
+			errorMsg.Valid = true
+
+			data.SetErrorMsg(errorMsg)
+			data.SetOperator(10000)
+			store.Save(data)
+			return
+		}
+
+		page++
+		querys.Set("page", strconv.Itoa(page))
+		wf.SetQuery(querys)
+
+		data.SetErrorMsg(errorMsg)
+		store.Save(data)
+
+		time.Sleep(time.Second * 2)
+	}
+}
+
+// OpenrecUser 获取用户信息
+type OpenrecUser struct {
+	hunter.PreCurlUrl
+}
+
+// Execute 执行方法
+func (or *OpenrecUser) Execute(cxt *hunter.TaskContext) {
+	resp, err := cxt.Hunt()
+	if err != nil {
+		log.Println(err)
+		return
+	}
+
+	resp.Content()
+}