add new nimo code

change more
2020-08-28 19:07:12 +08:00
parent eee4e30585
commit 5d50119825
30 changed files with 10540 additions and 110 deletions
--- a/tasks/nimo/nimo_task1/nimo_task1.go
+++ b/tasks/nimo/nimo_task1/nimo_task1.go
@@ -0,0 +1,106 @@
+package main
+
+import (
+	"database/sql"
+	"encoding/json"
+	"intimate"
+	"log"
+	"time"
+
+	"github.com/474420502/gcurl"
+	"github.com/tidwall/gjson"
+)
+
+// estore 解析存储连接实例
+var estore *intimate.StoreExtractor = intimate.NewStoreExtractor()
+
+// Execute 执行
+func Execute() {
+
+	nimoapi := `curl 'https://api.nimo.tv/oversea/nimo/api/v2/liveRoom/liveRoomPage-1-100-/HK/1028/1000' \
+	-H 'authority: api.nimo.tv' \
+	-H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36' \
+	-H 'content-type: multipart/form-data; boundary=----WebKitFormBoundary3bCA1lzvhj4kBR4Q' \
+	-H 'accept: */*' \
+	-H 'origin: https://www.nimo.tv' \
+	-H 'sec-fetch-site: same-site' \
+	-H 'sec-fetch-mode: cors' \
+	-H 'sec-fetch-dest: empty' \
+	-H 'referer: https://www.nimo.tv/lives' \
+	-H 'accept-language: zh-CN,zh;q=0.9' \
+	--data-binary $'------WebKitFormBoundary3bCA1lzvhj4kBR4Q\r\nContent-Disposition: form-data; name="keyType"\r\n\r\n0\r\n------WebKitFormBoundary3bCA1lzvhj4kBR4Q\r\nContent-Disposition: form-data; name="body"\r\n\r\n{"deviceType":7,"requestSource":"WEB","iNetType":5}\r\n------WebKitFormBoundary3bCA1lzvhj4kBR4Q--\r\n' \
+	--compressed`
+
+	curl := gcurl.Parse(nimoapi)
+	ses := curl.CreateSession()
+	tp := curl.CreateTemporary(ses)
+
+	param := tp.PathParam("liveRoomPage-(1)-")
+	ps := intimate.NewPerfectShutdown()
+	for !ps.IsClose() {
+
+		resp, err := tp.Execute()
+		if err != nil {
+			log.Println(err)
+			time.Sleep(time.Second)
+			continue
+		}
+
+		result := gjson.ParseBytes(resp.Content())
+		roomlist := result.Get("data.result.liveRoomViewList")
+
+		if !roomlist.IsArray() {
+			log.Println("json is error")
+			log.Println(string(resp.Content()))
+			break
+		}
+
+		rooms := roomlist.Array()
+		log.Println(tp.GetURLRawPath(), "rooms:", len(rooms))
+		if len(rooms) == 0 {
+			param.IntSet(1)
+			time.Sleep(time.Minute * 10)
+			continue
+		}
+
+		for _, room := range rooms {
+
+			streamer := &intimate.Streamer{}
+			streamer.Platform = intimate.PNimo
+
+			if userid := room.Get("id").String(); userid != "" {
+
+				streamer.UserId = userid
+				streamer.LiveUrl = sql.NullString{String: "https://www.nimo.tv/live/" + userid, Valid: true}
+
+				channel := room.Get("roomTypeName").String()
+				streamer.Channel = sql.NullString{String: channel, Valid: channel != ""}
+				username := room.Get("anchorName").String()
+				streamer.UserName = sql.NullString{String: username, Valid: username != ""}
+
+				if rtags := room.Get("anchorLabels"); rtags.IsArray() {
+
+					var tags []string
+					for _, r := range rtags.Array() {
+						tag := r.Get("labelName").String()
+						tags = append(tags, tag)
+					}
+
+					data, err := json.Marshal(tags)
+					if err != nil {
+						panic(err)
+					}
+					streamer.Tags = data
+
+				}
+
+				streamer.UpdateInterval = 120
+				// spew.Println(streamer)
+				estore.InsertStreamer(streamer)
+			} else {
+				log.Println("userid is null.", room.String())
+			}
+		}
+		param.IntAdd(1)
+	}
+}
--- a/tasks/nimo/nimo_task1/nimo_task1_test.go
+++ b/tasks/nimo/nimo_task1/nimo_task1_test.go
@@ -0,0 +1,55 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/474420502/gcurl"
+)
+
+func CrawlContent(args ...interface{}) []byte {
+	nimoapi := `curl 'https://api.nimo.tv/oversea/nimo/api/v2/liveRoom/liveRoomPage-%d-30-/HK/1028/1000' \
+	-H 'authority: api.nimo.tv' \
+	-H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36' \
+	-H 'content-type: multipart/form-data; boundary=----WebKitFormBoundary3bCA1lzvhj4kBR4Q' \
+	-H 'accept: */*' \
+	-H 'origin: https://www.nimo.tv' \
+	-H 'sec-fetch-site: same-site' \
+	-H 'sec-fetch-mode: cors' \
+	-H 'sec-fetch-dest: empty' \
+	-H 'referer: https://www.nimo.tv/lives' \
+	-H 'accept-language: zh-CN,zh;q=0.9' \
+	--data-binary $'------WebKitFormBoundary3bCA1lzvhj4kBR4Q\r\nContent-Disposition: form-data; name="keyType"\r\n\r\n0\r\n------WebKitFormBoundary3bCA1lzvhj4kBR4Q\r\nContent-Disposition: form-data; name="body"\r\n\r\n{"deviceType":7,"requestSource":"WEB","iNetType":5}\r\n------WebKitFormBoundary3bCA1lzvhj4kBR4Q--\r\n' \
+	--compressed`
+
+	curl := gcurl.Parse(fmt.Sprintf(nimoapi, 1))
+	tp := curl.CreateTemporary(nil)
+
+	resp, err := tp.Execute()
+	if err != nil {
+		panic(err)
+	}
+	return resp.Content()
+}
+
+func openTestFile(...interface{}) []byte {
+	f, err := os.Open("../../../testfile/nimo1.json")
+	if err != nil {
+		panic(err)
+	}
+	data, err := ioutil.ReadAll(f)
+	if err != nil {
+		panic(err)
+	}
+	return data
+}
+
+var Crawl func(...interface{}) []byte
+
+func Test(t *testing.T) {
+
+	Execute()
+
+}