From 925b7d42c760a853d9001d0808c7999f0f287e73 Mon Sep 17 00:00:00 2001 From: eson Date: Tue, 15 Sep 2020 17:09:22 +0800 Subject: [PATCH] add: mirrativ init --- .../mirrativ_extractor/mirrativ_extractor.go | 1 + .../mirrativ_extractor_test.go | 7 + extractor_field.go | 2 + platform_list.go | 3 + .../mirrativ/mirrativ_task1/mirrativ_task1.go | 81 +++++++++ .../mirrativ_task1/mirrativ_task1_test.go | 9 + testfile/mirrativ.json | 159 ++++++++++++++++++ 7 files changed, 262 insertions(+) create mode 100644 extractor/mirrativ_extractor/mirrativ_extractor.go create mode 100644 extractor/mirrativ_extractor/mirrativ_extractor_test.go create mode 100644 tasks/mirrativ/mirrativ_task1/mirrativ_task1.go create mode 100644 tasks/mirrativ/mirrativ_task1/mirrativ_task1_test.go create mode 100644 testfile/mirrativ.json diff --git a/extractor/mirrativ_extractor/mirrativ_extractor.go b/extractor/mirrativ_extractor/mirrativ_extractor.go new file mode 100644 index 0000000..06ab7d0 --- /dev/null +++ b/extractor/mirrativ_extractor/mirrativ_extractor.go @@ -0,0 +1 @@ +package main diff --git a/extractor/mirrativ_extractor/mirrativ_extractor_test.go b/extractor/mirrativ_extractor/mirrativ_extractor_test.go new file mode 100644 index 0000000..02d22fb --- /dev/null +++ b/extractor/mirrativ_extractor/mirrativ_extractor_test.go @@ -0,0 +1,7 @@ +package main + +import "testing" + +func TestDo(t *testing.T) { + +} diff --git a/extractor_field.go b/extractor_field.go index 4e1b691..e7577bf 100644 --- a/extractor_field.go +++ b/extractor_field.go @@ -46,6 +46,7 @@ type Streamer struct { Channel *sql.NullString `field:"channel"` // Tags interface{} `field:"tags"` Ext interface{} `field:"ext"` // + Comments interface{} `field:"comments"` IsUpdateStreamer bool // 更新上面的内容 IsUpdateUrl bool @@ -89,6 +90,7 @@ type CollectLog struct { Tags interface{} `field:"tags"` Ext interface{} `field:"ext"` // ErrorMsg *sql.NullString `field:"error_msg"` // + Comments interface{} `field:"comments"` // } // Get Simple Value diff --git a/platform_list.go b/platform_list.go index 98ef433..2006a1d 100644 --- a/platform_list.go +++ b/platform_list.go @@ -15,4 +15,7 @@ const ( // PNimo PNimo 平台 PNimo Platform = "nimo" + + // PMirrativ PNimo 平台 + PMirrativ Platform = "mirrativ" ) diff --git a/tasks/mirrativ/mirrativ_task1/mirrativ_task1.go b/tasks/mirrativ/mirrativ_task1/mirrativ_task1.go new file mode 100644 index 0000000..3bcb2f5 --- /dev/null +++ b/tasks/mirrativ/mirrativ_task1/mirrativ_task1.go @@ -0,0 +1,81 @@ +package main + +import ( + "database/sql" + "intimate" + "log" + "time" + + "github.com/474420502/gcurl" + "github.com/tidwall/gjson" +) + +func main() { + bcurl := `curl 'https://www.mirrativ.com/api/live/catalog?id=2&cursor=%s' \ + -H 'authority: www.mirrativ.com' \ + -H 'accept: application/json' \ + -H 'x-timezone: Asia/Shanghai' \ + -H 'x-csrf-token: F3Ojd6RBtApP6YAZzVn-9jWN1of159VxAqOQL1Zn' \ + -H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36' \ + -H 'content-type: application/json' \ + -H 'sec-fetch-site: same-origin' \ + -H 'sec-fetch-mode: cors' \ + -H 'sec-fetch-dest: empty' \ + -H 'referer: https://www.mirrativ.com/' \ + -H 'accept-language: zh-CN,zh;q=0.9,ja;q=0.8' \ + -H 'cookie: f=A2D75F0E-D218-11EA-A042-452BF6D21CE8; _ga=GA1.2.689947597.1596081392; mr_id=kxb65LddGMZf5C28jkR_tGCZD_ZFOAepD5gfXO7eNjfPMB8EKYvU1Vg_Y29V1lsa; _gid=GA1.2.2116692650.1600139685; lang=ja' \ + --compressed` + + curl := gcurl.Parse(bcurl) + tp := curl.CreateTemporary(nil) + cursor := tp.QueryParam(`cursor`) + cursor.StringSet("") + + for { + + log.Println(tp.ParsedURL.String()) + resp, err := tp.Execute() + if err != nil { + log.Println(err) + time.Sleep(time.Second * 2) + continue + } + + apijson := gjson.ParseBytes(resp.Content()) + next := apijson.Get("next_cursor").String() + + for _, liveinfo := range apijson.Get("list").Array() { + var prekey string + if liveinfo.Get("live_preview").Exists() { + prekey = "live_preview" + } else { + prekey = "live" + } + owner := liveinfo.Get(prekey + ".owner") + if guserid := owner.Get("user_id"); guserid.String() != "" { + streamer := &intimate.Streamer{} + streamer.Platform = intimate.PMirrativ + streamer.Operator = 0 + streamer.UserId = &guserid.Str + streamer.UserName = &sql.NullString{String: liveinfo.Get("name").String(), Valid: true} + streamer.UpdateInterval = 60 + streamer.UpdateTime = intimate.GetUpdateTimeNow() + + err = intimate.TStreamer.InsertOrUpdate( + streamer, + intimate.DUpdate{Field: "update_time"}, + ) + if err != nil { + log.Println(err) + panic(err) + } + } + } + if next == "" { + time.Sleep(time.Minute * 5) + } else { + time.Sleep(time.Second * 2) + } + cursor.StringSet(next) + } +} diff --git a/tasks/mirrativ/mirrativ_task1/mirrativ_task1_test.go b/tasks/mirrativ/mirrativ_task1/mirrativ_task1_test.go new file mode 100644 index 0000000..0ca53de --- /dev/null +++ b/tasks/mirrativ/mirrativ_task1/mirrativ_task1_test.go @@ -0,0 +1,9 @@ +package main + +import ( + "testing" +) + +func TestMain(t *testing.T) { + +} diff --git a/testfile/mirrativ.json b/testfile/mirrativ.json new file mode 100644 index 0000000..0fbad59 --- /dev/null +++ b/testfile/mirrativ.json @@ -0,0 +1,159 @@ +{ + "status": { + "msg": "", + "ok": 1, + "error": "", + "captcha_url": "", + "error_code": 0, + "message": "" + }, + "current_cursor": "MDoxMzplbjoyNjY2OTA2MS8w", + "next_cursor": "", + "latest_diamonds_given_at": null, + "announcement_urls": {}, + "list": [ + { + "live_preview": { + "is_private": 0, + "collab_supported": 1, + "sticker_enabled": 0, + "is_gift_supported": 0, + "collab_has_vacancy": 1, + "stamp_num": 0, + "live_id": "rAjAr5gwVtdkj1c-x8FNxA", + "collab_online_user_num": 0, + "share_image_url": "https://cdn.mirrativ.com/mirrorman-prod/image/live_share/6cae74e76b205f415051b28aaf06ef9bb5f08dd8e03f54678a449551698788bf_share.jpeg?1600143674", + "following_viewer_image_urls": [], + "is_mirrorable": 0, + "description": "", + "total_viewer_num": 18, + "ended_at": 0, + "thumbnail_image_url": "", + "is_archive": 0, + "online_user_num": 4, + "preview": { + "streaming_url_hls": "http://hls-cdn30.mirrativ.com/liveedge/db5968b68a5a11e46c624b9835d1b5902a221981e397a9c495a3a12bf6d58ac3/playlist.m3u8", + "streaming_url_edge": "rtmp://edge-210-140-191-152.mirrativ.com:1935/liveedge/db5968b68a5a11e46c624b9835d1b5902a221981e397a9c495a3a12bf6d58ac3" + }, + "title": "Iván say bullshit out loud. ", + "share_url": "https://www.mirrativ.com/live/rAjAr5gwVtdkj1c-x8FNxA", + "orientation": 2, + "app_icon_urls": [], + "max_online_viewer_num": 6, + "created_at": 1600055418, + "preview_blur_image_url": "", + "blur_image_url": "https://cdn.mirrativ.com/mirrorman-prod/image/live_capture/6cae74e76b205f415051b28aaf06ef9bb5f08dd8e03f54678a449551698788bf_m.jpeg?crop=16:9,offset-x0,offset-y26&blur=80&1600143674", + "started_at": 1600141392, + "is_live": 1, + "image_url_without_letterbox": "https://cdn.mirrativ.com/mirrorman-prod/image/live_capture/6cae74e76b205f415051b28aaf06ef9bb5f08dd8e03f54678a449551698788bf_m.jpeg?1600143674", + "thumbnail_blur_image_url": "", + "sticker_num": 0, + "thumbnail_frame": {}, + "max_collab_user_num": 3, + "comment_num": 0, + "joined_live_thumbnail_image_url": "https://cdn.mirrativ.com/mirrorman-prod/image/live_capture/6cae74e76b205f415051b28aaf06ef9bb5f08dd8e03f54678a449551698788bf_m.jpeg?1600143674", + "owner": { + "share_url": "https://www.mirrativ.com/user/10083948", + "profile_image_url": "https://cdn.mirrativ.com/mirrorman-prod/image/profile_image/58345863caafaf6f8272de423ce0b5b09375691a0e23c88a8080ecbe765467be_m.jpeg?1600055396", + "name": "Iván_rabbyt", + "description": "Ivan says bullshit out loud ", + "properties": [], + "badges": [ + { + "image_url": "https://cdn.mirrativ.com/assets/img/continuous_streamer/ic_badge_normal_holiday_L.en.png?v6", + "small_image_url": "https://cdn.mirrativ.com/assets/img/continuous_streamer/ic_badge_normal_holiday_S.en.png?v6" + } + ], + "is_continuous_streamer": 1, + "is_new": 0, + "catalog_label_image_url": "https://cdn.mirrativ.com/assets/img/continuous_streamer/ic_status_normal_holiday_en@3x.png?v5", + "user_id": "10083948", + "onlive": null + }, + "following_viewer_num": 0, + "sticker_display_type": 0, + "tags": [], + "is_following": 0, + "collab_enabled": 1, + "image_url": "https://cdn.mirrativ.com/mirrorman-prod/image/live_capture/6cae74e76b205f415051b28aaf06ef9bb5f08dd8e03f54678a449551698788bf_m.jpeg?crop=16:9,offset-x0,offset-y26&1600143674", + "bcsvr_key": "1abb7337:53JOIIhx", + "orientation_v2": 6, + "heartbeated_at": 1600143639 + }, + "type": "live_preview" + }, + { + "live": { + "is_private": 0, + "collab_supported": 0, + "sticker_enabled": 0, + "is_gift_supported": 1, + "collab_has_vacancy": 0, + "stamp_num": 0, + "live_id": "FV-79JspHmjsDLn4rdCJ3g", + "collab_online_user_num": 0, + "share_image_url": "https://cdn.mirrativ.com/mirrorman-prod/image/custom_thumbnail/80e27298548e6dc1df5691d3c1e120d04791f3de4e0cee8f3d01fadd11d13652_share.jpeg?1600143689", + "following_viewer_image_urls": [], + "is_mirrorable": 0, + "description": "", + "total_viewer_num": 103, + "ended_at": 0, + "thumbnail_image_url": "", + "is_archive": 0, + "online_user_num": 3, + "preview": { + "streaming_url_hls": "http://hls-cdn45.mirrativ.com/liveedge/2cd62d82d807effdd573e58d074ed31c40ab7a85a71332e729bc1dc86bf85938/playlist.m3u8", + "streaming_url_edge": "rtmp://edge-210-140-191-214.mirrativ.com:1935/liveedge/2cd62d82d807effdd573e58d074ed31c40ab7a85a71332e729bc1dc86bf85938" + }, + "title": "chilling and gaming", + "share_url": "https://www.mirrativ.com/live/FV-79JspHmjsDLn4rdCJ3g", + "app_id": "com.google.android.youtube", + "orientation": 0, + "app_icon_urls": [ + "https://cdn.mirrativ.com/mirrorman-prod/image/app_icon/c434fd240dfe93243c64944fa8cbfad0f3f373b6e380903f5d93ccf1c070a2d6.png" + ], + "max_online_viewer_num": 9, + "created_at": 1600132181, + "preview_blur_image_url": "", + "blur_image_url": "https://cdn.mirrativ.com/mirrorman-prod/image/thumbnail_image_url/80e27298548e6dc1df5691d3c1e120d04791f3de4e0cee8f3d01fadd11d13652_m.jpeg?blur=80&1600143689", + "started_at": 1600132190, + "is_live": 1, + "image_url_without_letterbox": "", + "thumbnail_blur_image_url": "", + "sticker_num": 0, + "thumbnail_frame": {}, + "comment_num": 11, + "max_collab_user_num": 1, + "joined_live_thumbnail_image_url": "https://cdn.mirrativ.com/mirrorman-prod/image/live_capture/80e27298548e6dc1df5691d3c1e120d04791f3de4e0cee8f3d01fadd11d13652_m.jpeg?1600143689", + "owner": { + "share_url": "https://www.mirrativ.com/user/3399944", + "profile_image_url": "https://cdn.mirrativ.com/mirrorman-prod/image/profile_image/3b3408709e40caf081e5f20d5a58dd3cb59b83e7d994b3f9119ca8f675a3bc9e_m.jpeg?1600121828", + "name": "Branndon", + "description": "Fuk fortnite", + "properties": [], + "badges": [ + { + "image_url": "https://cdn.mirrativ.com/assets/img/continuous_streamer/ic_badge_super_holiday_L.en.png?v6", + "small_image_url": "https://cdn.mirrativ.com/assets/img/continuous_streamer/ic_badge_super_holiday_S.en.png?v6" + } + ], + "is_continuous_streamer": 1, + "is_new": 0, + "catalog_label_image_url": "https://cdn.mirrativ.com/assets/img/continuous_streamer/ic_status_super_holiday_en@3x.png?v5", + "user_id": "3399944", + "onlive": null + }, + "following_viewer_num": 0, + "sticker_display_type": 0, + "tags": [], + "is_following": 0, + "collab_enabled": 0, + "image_url": "https://cdn.mirrativ.com/mirrorman-prod/image/thumbnail_image_url/80e27298548e6dc1df5691d3c1e120d04791f3de4e0cee8f3d01fadd11d13652_m.jpeg?1600143689", + "bcsvr_key": "1abd0717:FX5ZAqVh", + "orientation_v2": 0, + "heartbeated_at": 1600143665 + }, + "type": "live" + } + ] +} \ No newline at end of file