1. 修复 store_extractor Update 错误.

2. 重构前两个网站的代码架构不合理.
3. 减少冗余代码

tasks/openrec/openrec_task1/main.go

@@ -1,8 +1,5 @@
 package main
 
-import "github.com/474420502/hunter"
-
 func main() {
-	ht := hunter.NewHunter(openrecRanking)
-	ht.Execute()
+	Execute()
 }

tasks/openrec/openrec_task1/task_openrec.go

@@ -4,29 +4,24 @@ import (
 	"encoding/json"
 	"intimate"
 	"log"
-	"os"
-	"os/signal"
 	"strconv"
-	"sync/atomic"
-	"syscall"
 	"time"
 
-	"github.com/474420502/hunter"
+	"github.com/474420502/gcurl"
 	"github.com/tidwall/gjson"
 )
 
-var openrecRanking *OpenrecRanking
-
 // sstore  源存储实例, 为存储源数据的实现. 表格具体参考sql/intimate_source.sql
 var sstore *intimate.StoreSource = intimate.NewStoreSource(string(intimate.STOpenrec))
 
 // estore 解析存储连接实例
 var estore *intimate.StoreExtractor = intimate.NewStoreExtractor()
 
-func init() {
+// Execute 执行方法
+func Execute() {
 
-	openrecRanking = &OpenrecRanking{}
-	openrecRanking.PreCurlUrl = `curl 'https://public.openrec.tv/external/api/v5/channel-ranks?period=monthly&date=&tag=&page=1' \
+	ps := intimate.NewPerfectShutdown()
+	turl := `curl 'https://public.openrec.tv/external/api/v5/channel-ranks?period=monthly&date=&tag=&page=1' \
 	-H 'authority: public.openrec.tv' \
 	-H 'accept: application/json, text/javascript, */*; q=0.01' \
 	-H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36' \
@@ -39,43 +34,25 @@ func init() {
 	-H 'if-none-match: W/"25edb-aUYBdmLqZcr6DW4ZWKX9r2aqolg"' \
 	--compressed`
 
-}
+	g := gcurl.ParseRawCURL(turl)
+	tp := g.Temporary()
 
-// OpenrecRanking 获取排名任务
-type OpenrecRanking struct {
-	hunter.PreCurlUrl
-}
+	for !ps.IsClose() {
 
-// Execute 执行方法
-func (or *OpenrecRanking) Execute(cxt *hunter.TaskContext) {
-
-	var loop int32 = 1
-
-	go func() {
-		signalchan := make(chan os.Signal)
-		signal.Notify(signalchan, syscall.SIGKILL, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGSTOP)
-		log.Println("accept stop command:", <-signalchan)
-		atomic.StoreInt32(&loop, 0)
-	}()
-
-	for atomic.LoadInt32(&loop) > 0 {
-
-		resp, err := cxt.Hunt()
+		resp, err := tp.Execute()
 		if err != nil {
 			log.Println(err)
 			time.Sleep(time.Second * 2)
 			continue
 		}
 
-		tp := cxt.Temporary()
-
 		content := resp.Content()
 		if len(content) <= 200 { //末页时没有内容返回, 末页退出
 			finishpoint := time.Now()
 			log.Println("任务Ranking UserId结束休眠, 下次启动时间:", finishpoint.Add(time.Minute*120))
 			for time.Now().Sub(finishpoint) < time.Minute*120 {
 				time.Sleep(time.Second)
-				if atomic.LoadInt32(&loop) <= 0 {
+				if ps.IsClose() {
 					return
 				}
 			}
@@ -93,13 +70,6 @@ func (or *OpenrecRanking) Execute(cxt *hunter.TaskContext) {
 			for _, User := range result.Array() {
 
 				userid := User.Get("channel.id").String()
-
-				// data := &intimate.Source{}
-				// data.Source = sql.NullString{String: userid, Valid: len(userid) > 0}
-				// data.Url = tp.GetRawURL()
-				// data.TargetType = string(intimate.TTOpenrecUser)
-				// sstore.Insert(data)
-
 				streamer := &intimate.Streamer{}
 				streamer.UserId = userid
 				streamer.Platform = intimate.Popenrec
@@ -123,17 +93,7 @@ func (or *OpenrecRanking) Execute(cxt *hunter.TaskContext) {
 		}
 
 		// 修改url query 参数的page递增. 遍历所有页面
-		querys := tp.GetQuery()
-		page, err := strconv.Atoi(querys.Get("page"))
-		if err != nil {
-			log.Println(err)
-			return
-		}
-
-		page++
-		querys.Set("page", strconv.Itoa(page))
-		tp.SetQuery(querys)
-
+		tp.QueryParam("page").IntAdd(1)
 		time.Sleep(time.Second * 1)
 	}
 }

tasks/openrec/openrec_task1/task_openrec_test.go

@@ -63,7 +63,6 @@ func TestTimeAdd(t *testing.T) {
 	t.Error(time.Now().Sub(finishpoint) > time.Second*1)
 }
 
-func TestRankingInsert(t *testing.T) {
-	ht := hunter.NewHunter(openrecRanking)
-	ht.Execute()
+func TestMain(t *testing.T) {
+	main()
 }