1. 修复 store_extractor Update 错误.

2. 重构前两个网站的代码架构不合理.
3. 减少冗余代码

tasks/openrec/openrec_task2/main.go

@@ -1,8 +1,5 @@
 package main
 
-import "github.com/474420502/hunter"
-
 func main() {
-	ht := hunter.NewHunter(oer)
-	ht.Execute()
+	Execute()
 }

tasks/openrec/openrec_task2/task_openrec.go

@@ -5,21 +5,13 @@ import (
 	"encoding/json"
 	"intimate"
 	"log"
-	"os"
-	"os/signal"
-	"strconv"
-	"sync/atomic"
-	"syscall"
 	"time"
 
 	"github.com/474420502/gcurl"
+	"github.com/474420502/requests"
 	"github.com/tidwall/gjson"
-
-	"github.com/474420502/hunter"
 )
 
-var oer *OpenrecExtratorRanking
-
 // sstore  源存储实例, 为存储源数据的实现. 表格具体参考sql/intimate_source.sql
 var sstore *intimate.StoreSource = intimate.NewStoreSource(string(intimate.STOpenrec))
 
@@ -27,29 +19,18 @@ var sstore *intimate.StoreSource = intimate.NewStoreSource(string(intimate.STOpe
 var estore *intimate.StoreExtractor = intimate.NewStoreExtractor()
 
 func init() {
-	oer = &OpenrecExtratorRanking{}
-}
 
-// OpenrecExtratorRanking 获取用户信息
-type OpenrecExtratorRanking struct {
-	// Store *intimate.Store
 }
 
 // Execute 执行方法
-func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) {
+func Execute() {
 
-	var loop int32 = 1
-
-	go func() {
-		signalchan := make(chan os.Signal)
-		signal.Notify(signalchan, syscall.SIGKILL, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGSTOP)
-		log.Println("accept stop command:", <-signalchan)
-		atomic.StoreInt32(&loop, 0)
-	}()
+	ps := intimate.NewPerfectShutdown()
+	ses := requests.NewSession()
 
 	var lasterr error = nil
 
-	for atomic.LoadInt32(&loop) > 0 {
+	for !ps.IsClose() {
 
 		streamer, err := estore.Pop(intimate.Popenrec) //队列里弹出一个streamer行. 进行解析
 
@@ -74,7 +55,8 @@ func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) {
 		// Check Userid
 
 		userUrl := updateUrl["user"]
-		tp := cxt.Session().Get(userUrl) // 获取user url页面数据
+		log.Println(userUrl)
+		tp := ses.Get(userUrl) // 获取user url页面数据
 		resp, err := tp.Execute()
 		streamer.UpdateTime = sql.NullTime{Time: time.Now(), Valid: true}
 
@@ -84,7 +66,7 @@ func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) {
 			continue
 		}
 
-		cookies := cxt.Session().GetCookies(tp.GetParsedURL())
+		cookies := ses.GetCookies(tp.GetParsedURL())
 
 		scurl := updateUrl["supporters"] //获取打赏者的数据
 		curl := gcurl.ParseRawCURL(scurl)
@@ -128,16 +110,17 @@ func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) {
 			}
 			supporters = append(supporters, string(resp.Content()))
 
-			page := supportersQuery.Get("page_number") // page_number 加1
-			pageint, err := strconv.Atoi(page)
-			if err != nil {
-				log.Println(err)
-				break
-			}
-			pageint++
-			page = strconv.Itoa(pageint)
-			supportersQuery.Set("page_number", page)
-			temporary.SetQuery(supportersQuery)
+			temporary.QueryParam("page_number").IntAdd(1)
+			// page := supportersQuery.Get("page_number") // page_number 加1
+			// pageint, err := strconv.Atoi(page)
+			// if err != nil {
+			// 	log.Println(err)
+			// 	break
+			// }
+			// pageint++
+			// page = strconv.Itoa(pageint)
+			// supportersQuery.Set("page_number", page)
+			// temporary.SetQuery(supportersQuery)
 		}
 
 		// cookies := cxt.Session().GetCookies(wf.GetParsedURL())
@@ -147,7 +130,7 @@ func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) {
 		ext["html_user"] = string(resp.Content())
 
 		liveUrl := updateUrl["live"]
-		tp = cxt.Session().Get(liveUrl)
+		tp = ses.Get(liveUrl)
 		resp, err = tp.Execute()
 		if err != nil {
 			log.Println(err)
@@ -164,7 +147,10 @@ func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) {
 			continue
 		}
 
-		streamer.Operator = int32(intimate.OperatorOK)
+		// streamer.Platform = intimate.Popenrec
+		streamer.UpdateInterval = 120
+		streamer.UpdateTime = sql.NullTime{Time: time.Now(), Valid: true}
+		streamer.Operator = 0
 
 		source := &intimate.Source{}
 		source.Target = intimate.TOpenrecUser
@@ -172,7 +158,7 @@ func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) {
 		source.StreamerId = sql.NullInt64{Int64: streamer.Uid, Valid: true}
 		sstore.Insert(source)
 
-		estore.UpdateOperator(streamer)
+		estore.UpdateStreamer(streamer)
 	}
 
 }

tasks/openrec/openrec_task2/task_openrec_test.go

@@ -2,11 +2,8 @@ package main
 
 import (
 	"testing"
-
-	"github.com/474420502/hunter"
 )
 
-func TestOpenrecUser(t *testing.T) {
-	ht := hunter.NewHunter(oer)
-	ht.Execute()
+func TestMain(t *testing.T) {
+	main()
 }