完成openrec源的获取.

.gitignore

@@ -4,5 +4,6 @@ screenlog.*
 *.7z
 intimate
 *.gz
+debug.test
 
 

sql/intimate_extractor.sql

@@ -6,7 +6,7 @@ CREATE TABLE IF NOT EXISTS `anchor_info` (
   `platform`      varchar(255) NOT NULL,
   `anchor_id`     varchar(255) NOT NULL,
   `anchor_name`   varchar(255) NOT NULL,
-  `platform_url`  text NOT NULL,
+  `live_url`      text,
   `channel`       varchar(128) DEFAULT NULL,
   `show_type`     varchar(255) DEFAULT NULL,
   `update_time`   timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,

tasks/openrec/openrec_task1/task_openrec.go

@@ -8,9 +8,11 @@ import (
 	"time"
 
 	"github.com/474420502/hunter"
+	"github.com/tidwall/gjson"
 )
 
 var targetTypeRanking = "openrec_ranking"
+var targetTypeUser = "openrec_user"
 var openrecRanking *OpenrecRanking
 
 // store  源存储实例, 为存储源数据的实现. 表格具体参考sql/intimate_source.sql
@@ -44,7 +46,6 @@ func (or *OpenrecRanking) Execute(cxt *hunter.TaskContext) {
 
 	for {
 
-		errorMsg := sql.NullString{Valid: false}
 		resp, err := cxt.Hunt()
 		if err != nil {
 			log.Println(err)
@@ -53,36 +54,34 @@ func (or *OpenrecRanking) Execute(cxt *hunter.TaskContext) {
 
 		wf := cxt.Workflow()
 
-		data := &intimate.Source{}
 		content := resp.Content()
 		if len(content) <= 200 {
 			return
 		}
 
-		data.SetSource(sql.NullString{String: string(content), Valid: len(content) > 0})
+		result := gjson.ParseBytes(content)
-		data.SetUrl(wf.GetRawURL())
+		if result.IsArray() {
-		data.SetTargetType(targetTypeRanking)
+			for _, User := range result.Array() {
+				data := &intimate.Source{}
+				userid := User.Get("channel.id").String()
+
+				data.SetSource(sql.NullString{String: userid, Valid: len(userid) > 0})
+				data.SetUrl(wf.GetRawURL())
+				data.SetTargetType(targetTypeUser)
+				store.Insert(data)
+			}
+		}
 
 		querys := wf.GetQuery()
 		page, err := strconv.Atoi(querys.Get("page"))
 		if err != nil {
 			log.Println(err)
-			errorMsg.String = err.Error()
-			errorMsg.Valid = true
-
-			data.SetErrorMsg(errorMsg)
-			data.SetOperator(10000)
-			store.Insert(data)
 			return
 		}
 
 		page++
 		querys.Set("page", strconv.Itoa(page))
 		wf.SetQuery(querys)
-
-		data.SetErrorMsg(errorMsg)
-		store.Insert(data)
-
 		time.Sleep(time.Second * 2)
 	}
 }

tasks/openrec/openrec_task2/task_openrec.go

@@ -8,7 +8,6 @@ import (
 	"time"
 
 	"github.com/474420502/hunter"
-	"github.com/tidwall/gjson"
 )
 
 var targetTypeUser = "openrec_user"
@@ -32,7 +31,7 @@ func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) {
 
 	for {
 
-		source, err := store.Pop(targetTypeRanking)
+		source, err := store.Pop(targetTypeUser)
 		if err != nil {
 			log.Println(err)
 			return
@@ -42,52 +41,59 @@ func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) {
 			return
 		}
 
-		result := gjson.Parse(source.GetSource().String)
+		userSource := &intimate.Source{}
-		if result.IsArray() {
+		userid := source.GetSource().String
-			for _, User := range result.Array() {
+		userUrl := "https://www.openrec.tv/user/" + userid
-				userid := User.Get("channel.id").String()
+		userSource.SetUrl(userUrl)
-				ext := make(map[string]interface{})
 
-				wf := cxt.Session().Get("https://www.openrec.tv/user/" + userid)
+		wf := cxt.Session().Get(userUrl)
-				resp, err := wf.Execute()
+		resp, err := wf.Execute()
-				source.SetUpdateTime(time.Now())
+		source.SetUpdateTime(time.Now())
 
-				if err != nil {
+		if err != nil {
-					log.Println(err)
+			log.Println(err)
 
-					source.SetOperator(int32(intimate.OperatorError))
+			source.SetOperator(int32(intimate.OperatorError))
-					source.SetErrorMsg(sql.NullString{String: err.Error(), Valid: true})
+			source.SetErrorMsg(sql.NullString{String: err.Error(), Valid: true})
-					continue
+			continue
-				}
-
-				ext["user"] = string(resp.Content())
-
-				wf = cxt.Session().Get("https://www.openrec.tv/user/" + userid + "/supporters")
-				resp, err = wf.Execute()
-				if err != nil {
-					log.Println(err)
-					source.SetOperator(int32(intimate.OperatorError))
-					source.SetErrorMsg(sql.NullString{String: err.Error(), Valid: true})
-					continue
-				}
-				ext["user_supporters"] = string(resp.Content())
-
-				extJsonBytes, err := json.Marshal(ext)
-				if err != nil {
-					log.Println(err)
-					source.SetOperator(int32(intimate.OperatorError))
-					source.SetErrorMsg(sql.NullString{String: err.Error(), Valid: true})
-					continue
-				}
-
-				source.SetOperator(int32(intimate.OperatorOK))
-				source.SetExt(string(extJsonBytes))
-
-				store.Update(source)
-			}
-		} else {
-			log.Println("array error:", result.Str)
 		}
 
+		ext := make(map[string]interface{})
+
+		ext["user"] = string(resp.Content())
+
+		wf = cxt.Session().Get(userUrl + "/supporters")
+		resp, err = wf.Execute()
+		if err != nil {
+			log.Println(err)
+			source.SetOperator(int32(intimate.OperatorError))
+			source.SetErrorMsg(sql.NullString{String: err.Error(), Valid: true})
+			continue
+		}
+		ext["user_supporters"] = string(resp.Content())
+
+		wf = cxt.Session().Get("https://www.openrec.tv/live/" + userid)
+		resp, err = wf.Execute()
+		if err != nil {
+			log.Println(err)
+			source.SetOperator(int32(intimate.OperatorError))
+			source.SetErrorMsg(sql.NullString{String: err.Error(), Valid: true})
+			continue
+		}
+		ext["user_live"] = string(resp.Content())
+
+		extJsonBytes, err := json.Marshal(ext)
+		if err != nil {
+			log.Println(err)
+			source.SetOperator(int32(intimate.OperatorError))
+			source.SetErrorMsg(sql.NullString{String: err.Error(), Valid: true})
+			continue
+		}
+
+		source.SetOperator(int32(intimate.OperatorOK))
+		source.SetExt(string(extJsonBytes))
+		store.Update(source)
+
 	}
+
 }