add new nimo code

change more

tasks/nimo/nimo_task1/nimo_task1.go

@@ -0,0 +1,106 @@
+package main
+
+import (
+	"database/sql"
+	"encoding/json"
+	"intimate"
+	"log"
+	"time"
+
+	"github.com/474420502/gcurl"
+	"github.com/tidwall/gjson"
+)
+
+// estore 解析存储连接实例
+var estore *intimate.StoreExtractor = intimate.NewStoreExtractor()
+
+// Execute 执行
+func Execute() {
+
+	nimoapi := `curl 'https://api.nimo.tv/oversea/nimo/api/v2/liveRoom/liveRoomPage-1-100-/HK/1028/1000' \
+	-H 'authority: api.nimo.tv' \
+	-H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36' \
+	-H 'content-type: multipart/form-data; boundary=----WebKitFormBoundary3bCA1lzvhj4kBR4Q' \
+	-H 'accept: */*' \
+	-H 'origin: https://www.nimo.tv' \
+	-H 'sec-fetch-site: same-site' \
+	-H 'sec-fetch-mode: cors' \
+	-H 'sec-fetch-dest: empty' \
+	-H 'referer: https://www.nimo.tv/lives' \
+	-H 'accept-language: zh-CN,zh;q=0.9' \
+	--data-binary $'------WebKitFormBoundary3bCA1lzvhj4kBR4Q\r\nContent-Disposition: form-data; name="keyType"\r\n\r\n0\r\n------WebKitFormBoundary3bCA1lzvhj4kBR4Q\r\nContent-Disposition: form-data; name="body"\r\n\r\n{"deviceType":7,"requestSource":"WEB","iNetType":5}\r\n------WebKitFormBoundary3bCA1lzvhj4kBR4Q--\r\n' \
+	--compressed`
+
+	curl := gcurl.Parse(nimoapi)
+	ses := curl.CreateSession()
+	tp := curl.CreateTemporary(ses)
+
+	param := tp.PathParam("liveRoomPage-(1)-")
+	ps := intimate.NewPerfectShutdown()
+	for !ps.IsClose() {
+
+		resp, err := tp.Execute()
+		if err != nil {
+			log.Println(err)
+			time.Sleep(time.Second)
+			continue
+		}
+
+		result := gjson.ParseBytes(resp.Content())
+		roomlist := result.Get("data.result.liveRoomViewList")
+
+		if !roomlist.IsArray() {
+			log.Println("json is error")
+			log.Println(string(resp.Content()))
+			break
+		}
+
+		rooms := roomlist.Array()
+		log.Println(tp.GetURLRawPath(), "rooms:", len(rooms))
+		if len(rooms) == 0 {
+			param.IntSet(1)
+			time.Sleep(time.Minute * 10)
+			continue
+		}
+
+		for _, room := range rooms {
+
+			streamer := &intimate.Streamer{}
+			streamer.Platform = intimate.PNimo
+
+			if userid := room.Get("id").String(); userid != "" {
+
+				streamer.UserId = userid
+				streamer.LiveUrl = sql.NullString{String: "https://www.nimo.tv/live/" + userid, Valid: true}
+
+				channel := room.Get("roomTypeName").String()
+				streamer.Channel = sql.NullString{String: channel, Valid: channel != ""}
+				username := room.Get("anchorName").String()
+				streamer.UserName = sql.NullString{String: username, Valid: username != ""}
+
+				if rtags := room.Get("anchorLabels"); rtags.IsArray() {
+
+					var tags []string
+					for _, r := range rtags.Array() {
+						tag := r.Get("labelName").String()
+						tags = append(tags, tag)
+					}
+
+					data, err := json.Marshal(tags)
+					if err != nil {
+						panic(err)
+					}
+					streamer.Tags = data
+
+				}
+
+				streamer.UpdateInterval = 120
+				// spew.Println(streamer)
+				estore.InsertStreamer(streamer)
+			} else {
+				log.Println("userid is null.", room.String())
+			}
+		}
+		param.IntAdd(1)
+	}
+}

tasks/nimo/nimo_task1/nimo_task1_test.go

@@ -0,0 +1,55 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/474420502/gcurl"
+)
+
+func CrawlContent(args ...interface{}) []byte {
+	nimoapi := `curl 'https://api.nimo.tv/oversea/nimo/api/v2/liveRoom/liveRoomPage-%d-30-/HK/1028/1000' \
+	-H 'authority: api.nimo.tv' \
+	-H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36' \
+	-H 'content-type: multipart/form-data; boundary=----WebKitFormBoundary3bCA1lzvhj4kBR4Q' \
+	-H 'accept: */*' \
+	-H 'origin: https://www.nimo.tv' \
+	-H 'sec-fetch-site: same-site' \
+	-H 'sec-fetch-mode: cors' \
+	-H 'sec-fetch-dest: empty' \
+	-H 'referer: https://www.nimo.tv/lives' \
+	-H 'accept-language: zh-CN,zh;q=0.9' \
+	--data-binary $'------WebKitFormBoundary3bCA1lzvhj4kBR4Q\r\nContent-Disposition: form-data; name="keyType"\r\n\r\n0\r\n------WebKitFormBoundary3bCA1lzvhj4kBR4Q\r\nContent-Disposition: form-data; name="body"\r\n\r\n{"deviceType":7,"requestSource":"WEB","iNetType":5}\r\n------WebKitFormBoundary3bCA1lzvhj4kBR4Q--\r\n' \
+	--compressed`
+
+	curl := gcurl.Parse(fmt.Sprintf(nimoapi, 1))
+	tp := curl.CreateTemporary(nil)
+
+	resp, err := tp.Execute()
+	if err != nil {
+		panic(err)
+	}
+	return resp.Content()
+}
+
+func openTestFile(...interface{}) []byte {
+	f, err := os.Open("../../../testfile/nimo1.json")
+	if err != nil {
+		panic(err)
+	}
+	data, err := ioutil.ReadAll(f)
+	if err != nil {
+		panic(err)
+	}
+	return data
+}
+
+var Crawl func(...interface{}) []byte
+
+func Test(t *testing.T) {
+
+	Execute()
+
+}

tasks/openrec/openrec_task1/task_openrec.go

@@ -34,7 +34,7 @@ func Execute() {
 	-H 'if-none-match: W/"25edb-aUYBdmLqZcr6DW4ZWKX9r2aqolg"' \
 	--compressed`
 
-	g := gcurl.ParseRawCURL(turl)
+	g := gcurl.Parse(turl)
 	tp := g.Temporary()
 
 	for !ps.IsClose() {

tasks/openrec/openrec_task2/task_openrec.go

@@ -69,7 +69,7 @@ func Execute() {
 		cookies := ses.GetCookies(tp.GetParsedURL())
 
 		scurl := updateUrl["supporters"] //获取打赏者的数据
-		curl := gcurl.ParseRawCURL(scurl)
+		curl := gcurl.Parse(scurl)
 		supportersSession := curl.CreateSession()
 
 		temporary := curl.CreateTemporary(supportersSession)

tasks/twitch/twitch_task1/task_twitch_test.go

@@ -4,7 +4,7 @@ import (
 	"testing"
 )
 
-func estCase1(t *testing.T) {
+func TestCase1(t *testing.T) {
 	Execute()
 }
 

tasks/twitch/twitch_task2/task_twitch.go

@@ -159,7 +159,9 @@ func Execute() {
 			streamer.Operator = 0
 			if estore.InsertStreamer(streamer) {
 				// log.Println("streamer update tags", streamer.Uid, tags)
-				estore.Update(streamer, "Tags", streamer.Tags)
+				if streamer.Tags != nil {
+					estore.Update(streamer, "Tags", streamer.Tags)
+				}
 			}
 		}
 		log.Println("streamer find", len(articles))

tasks/twitch/twitch_task2/task_twitch_test.go

@@ -2,6 +2,6 @@ package main
 
 import "testing"
 
-func estMain(t *testing.T) {
+func TestMain(t *testing.T) {
 	main()
 }