From fb3b09fc387551b5ba6017bdb06b2fb3109cadae Mon Sep 17 00:00:00 2001 From: eson Date: Wed, 16 Sep 2020 18:53:26 +0800 Subject: [PATCH] mirrativ finish 90% --- autostore.go | 2 +- .../mirrativ_extractor_test.go | 118 +++++++++- extractor_field.go | 2 +- .../mirrativ/mirrativ_task1/mirrativ_task1.go | 7 +- .../mirrativ_task1/mirrativ_task1_test.go | 2 +- testfile/mirrativ.json | 203 +++++++++++++++++- 6 files changed, 320 insertions(+), 14 deletions(-) diff --git a/autostore.go b/autostore.go index a7c73c8..6812888 100644 --- a/autostore.go +++ b/autostore.go @@ -145,7 +145,7 @@ func (queue *Queue) Pop() (result interface{}, err error) { rows, err := tx.Query(selectsql, queue.cond.CondArgs...) if err != nil { - return nil, fmt.Errorf("table: %s queue is empty", queue.table.name) + return nil, fmt.Errorf("table: %s queue is empty. %s", queue.table.name, err.Error()) } var fields = make([]interface{}, len(queue.fieldIndex)) diff --git a/extractor/mirrativ_extractor/mirrativ_extractor_test.go b/extractor/mirrativ_extractor/mirrativ_extractor_test.go index 49c9507..acc2abd 100644 --- a/extractor/mirrativ_extractor/mirrativ_extractor_test.go +++ b/extractor/mirrativ_extractor/mirrativ_extractor_test.go @@ -1,22 +1,39 @@ package main import ( + "database/sql" + "encoding/json" "intimate" "log" "testing" "time" "github.com/474420502/gcurl" + "github.com/tidwall/gjson" ) func main() { + sessionstr := ` + -H 'authority: www.mirrativ.com' + -H 'accept: application/json' + -H 'x-timezone: Asia/Shanghai' + -H 'x-csrf-token: F3Ojd6RBtApP6YAZzVn-9jWN1of159VxAqOQL1Zn' + -H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36' + -H 'content-type: application/json' + -H 'sec-fetch-site: same-origin' + -H 'sec-fetch-mode: cors' + -H 'sec-fetch-dest: empty' + -H 'referer: https://www.mirrativ.com/live/O5Ia4iX9c5CeZj7DFtg52Q' + -H 'accept-language: zh-CN,zh;q=0.9,ja;q=0.8' + -H 'cookie: f=A2D75F0E-D218-11EA-A042-452BF6D21CE8; _ga=GA1.2.689947597.1596081392; mr_id=kxb65LddGMZf5C28jkR_tGCZD_ZFOAepD5gfXO7eNjfPMB8EKYvU1Vg_Y29V1lsa; _gid=GA1.2.2116692650.1600139685; lang=ja'` + ps := intimate.NewPerfectShutdown() - gprofile := gcurl.Parse(`https://www.mirrativ.com/api/user/profile?user_id=103383701`) + gprofile := gcurl.Parse(`curl 'https://www.mirrativ.com/api/user/profile?user_id=103383701'` + sessionstr) tpProfile := gprofile.CreateTemporary(nil) tpProfileUserID := tpProfile.QueryParam("user_id") - g := gcurl.Parse(`https://www.mirrativ.com/api/live/live?live_id=O5Ia4iX9c5CeZj7DFtg52Q`) + g := gcurl.Parse(`culr 'https://www.mirrativ.com/api/live/live?live_id=O5Ia4iX9c5CeZj7DFtg52Q'` + sessionstr) tpLive := g.CreateTemporary(nil) tpLiveID := tpLive.QueryParam("live_id") @@ -28,9 +45,11 @@ func main() { time.Sleep(time.Second * 2) continue } - + now := &sql.NullTime{Time: time.Now(), Valid: true} streamer := istreamer.(*intimate.Streamer) + streamer.UpdateTime = now userid := *streamer.UserId + log.Println(userid) tpProfileUserID.StringSet(userid) resp, err := tpProfile.Execute() @@ -40,10 +59,99 @@ func main() { continue } - profilejson := gcurl.Parse(string(resp.Content())) + clog := &intimate.CollectLog{} + clog.Platform = intimate.PMirrativ + clog.UpdateTime = now + clog.UserId = userid + clog.StreamerUid = streamer.Uid + profilejson := gjson.ParseBytes(resp.Content()) + if result := profilejson.Get("follower_num"); result.Exists() { + clog.Followers = &sql.NullInt64{Int64: result.Int(), Valid: true} + } + + if result := profilejson.Get("onlive.live_id"); result.Exists() { + liveID := result.String() + tpLiveID.StringSet(liveID) + resp, err = tpLive.Execute() + if err != nil { + log.Println(err) + continue + } + + livejson := gjson.ParseBytes(resp.Content()) + if result := livejson.Get("total_viewer_num"); result.Exists() { + clog.Views = &sql.NullInt64{Int64: result.Int(), Valid: true} + } + + if result := livejson.Get("title"); result.Exists() { + clog.LiveTitle = &sql.NullString{String: result.String(), Valid: true} + } + + if result := livejson.Get("started_at"); result.Exists() { + clog.LiveStartTime = &sql.NullTime{Time: time.Unix(result.Int(), 0), Valid: true} + } + + if result := livejson.Get("heartbeated_at"); result.Exists() { + clog.LiveEndTime = &sql.NullTime{Time: time.Unix(result.Int(), 0), Valid: true} + } + + if result := livejson.Get("app_id"); result.Exists() { + streamer.Channel = &sql.NullString{String: result.String(), Valid: true} + } + + if result := livejson.Get("timeline.#.app.short_title"); result.Exists() { + for _, tl := range result.Array() { + var tags []string = []string{tl.String()} + jtags, _ := json.Marshal(tags) + streamer.Tags = jtags + clog.Tags = jtags + break + } + } else { + log.Println(string(resp.Content())) + return + } + + if result := livejson.Get("gift_ranking_url"); result.Exists() { + // streamer.Channel = &sql.NullString{String: result.String(), Valid: true} + gifturl := "curl '" + result.String() + "&type=monthly&cursor='" + sessionstr + ggift := gcurl.Parse(gifturl) + tp := ggift.CreateTemporary(nil) + tp.SetURLRawPath("/api/gift/ranking") + pcursor := tp.QueryParam("cursor") + var gratuity int64 = 0 + + for { + giftdata, err := tp.Execute() + giftjson := gjson.ParseBytes(giftdata.Content()) + if err != nil { + log.Println(err) + } else { + for _, rpoint := range giftjson.Get("ranking.#.point").Array() { + gratuity += rpoint.Int() + } + } + ncursor := giftjson.Get("next_cursor").String() + if ncursor == "" { + break + } + pcursor.StringSet(ncursor) + } + // https://www.mirrativ.com/gift/ranking?live_id=O5Ia4iX9c5CeZj7DFtg52Q&obfuscated_user_id=PgIBEgc6jVc + clog.Gratuity = &sql.NullInt64{Int64: gratuity, Valid: true} + } + + cid, err := intimate.TClog.InsertRetAutoID(clog) + if err != nil { + log.Println(err) + } + streamer.LatestLogUid = cid + } + intimate.TStreamer.Update(streamer) + + time.Sleep(time.Second * 2) } - } func TestDo(t *testing.T) { diff --git a/extractor_field.go b/extractor_field.go index e7577bf..c38962d 100644 --- a/extractor_field.go +++ b/extractor_field.go @@ -46,7 +46,7 @@ type Streamer struct { Channel *sql.NullString `field:"channel"` // Tags interface{} `field:"tags"` Ext interface{} `field:"ext"` // - Comments interface{} `field:"comments"` + // Comments interface{} `field:"comments"` IsUpdateStreamer bool // 更新上面的内容 IsUpdateUrl bool diff --git a/tasks/mirrativ/mirrativ_task1/mirrativ_task1.go b/tasks/mirrativ/mirrativ_task1/mirrativ_task1.go index 3bcb2f5..5fd7146 100644 --- a/tasks/mirrativ/mirrativ_task1/mirrativ_task1.go +++ b/tasks/mirrativ/mirrativ_task1/mirrativ_task1.go @@ -57,14 +57,11 @@ func main() { streamer.Platform = intimate.PMirrativ streamer.Operator = 0 streamer.UserId = &guserid.Str - streamer.UserName = &sql.NullString{String: liveinfo.Get("name").String(), Valid: true} + streamer.UserName = &sql.NullString{String: owner.Get("name").String(), Valid: true} streamer.UpdateInterval = 60 streamer.UpdateTime = intimate.GetUpdateTimeNow() - err = intimate.TStreamer.InsertOrUpdate( - streamer, - intimate.DUpdate{Field: "update_time"}, - ) + err = intimate.TStreamer.Insert(streamer) if err != nil { log.Println(err) panic(err) diff --git a/tasks/mirrativ/mirrativ_task1/mirrativ_task1_test.go b/tasks/mirrativ/mirrativ_task1/mirrativ_task1_test.go index 0ca53de..80820ef 100644 --- a/tasks/mirrativ/mirrativ_task1/mirrativ_task1_test.go +++ b/tasks/mirrativ/mirrativ_task1/mirrativ_task1_test.go @@ -5,5 +5,5 @@ import ( ) func TestMain(t *testing.T) { - + main() } diff --git a/testfile/mirrativ.json b/testfile/mirrativ.json index 2106e78..87da6fc 100644 --- a/testfile/mirrativ.json +++ b/testfile/mirrativ.json @@ -2361,4 +2361,205 @@ "collab_enabled": 1, "image_url": "https://cdn.mirrativ.com/mirrorman-prod/image/thumbnail_image_url/9bfba1b91a486106882577717aab4ccb4113b200d07dfb7127b087e4e52d8869_m.jpeg?1600164645", "orientation_v2": "6" -} \ No newline at end of file +} + + +{ + "obfuscated_user_id": "PgIBEgc6jVc", + "current_user_rank": null, + "ranking": [ + { + "point": 10, + "user": { + "share_url": "https://www.mirrativ.com/user/105481237", + "profile_image_url": "https://cdn.mirrativ.com/mirrorman-prod/image/profile_image/2d16160a4158c99daf92f287313a7b4df7797b9f1e07ae0f8983df42fbecec97_m.jpeg?1600205857", + "name": "wwwww", + "description": "", + "properties": [], + "badges": [], + "is_continuous_streamer": 0, + "is_new": 0, + "user_id": "105481237", + "onlive": null + } + }, + { + "point": 1, + "user": { + "share_url": "https://www.mirrativ.com/user/103544585", + "profile_image_url": "https://cdn.mirrativ.com/mirrorman-prod/image/profile_image/40cbd7501613cf08f45c56704b1a80796413d7ac0f9e06851d7a4ebe6f1fe7de_m.jpeg?1600217414", + "name": "ꗼ𝒓𝑦𝒖ꗼ", + "description": "俺のファンか仲良い人にしかギフトとかはあげません！まぁ俺にファンなんて居ないけど", + "properties": [], + "badges": [ + { + "image_url": "https://cdn.mirrativ.com/assets/img/continuous_streamer/ic_badge_master_holiday_L.en.png?v1", + "small_image_url": "https://cdn.mirrativ.com/assets/img/continuous_streamer/ic_badge_master_holiday_S.en.png?v1" + } + ], + "is_continuous_streamer": 1, + "is_new": 0, + "catalog_label_image_url": "https://cdn.mirrativ.com/assets/img/continuous_streamer/ic_status_master_holiday_en@3x.png?v5", + "user_id": "103544585", + "onlive": null + } + } + ], + "status": { + "msg": "", + "ok": 1, + "error": "", + "captcha_url": "", + "error_code": 0, + "message": "" + }, + "next_cursor": "", + "current_cursor": "bW9udGhseToxMDYwNjk5MTY6MjY2NzA0NzMvMA" +} + +{ + "streaming_url_hls": "http://hls-cdn22.mirrativ.com/liveedge/c4d32ce6edec9a37e7b20cff4b656a95714b5b9ee872964d9e435debe74a9518/playlist.m3u8", + "is_gift_supported": 1, + "live_id": "9LtbLw58uSrzig_NTxutIw", + "is_mirrorable": 0, + "description": "いつも拝聴ありがとうございます！\nクラッカーは配信が落ちてしまうので、禁止でお願い致しますm(_ _)m\n", + "total_viewer_num": 1127, + "thumbnail_image_url": "", + "is_archive": 0, + "is_singing_karaoke": 0, + "title": "雑談しながらゲームする！", + "max_online_viewer_num": 212, + "created_at": 1600194426, + "is_live": 1, + "started_at": 1600242640, + "blur_image_url": "https://cdn.mirrativ.com/mirrorman-prod/image/thumbnail_image_url/64c45462a6dc1113cf984a2bebac27d99cbb9b12c736ee3d104dd4c0c4e086db_m.jpeg?blur=80&1600244617", + "preview_blur_image_url": "", + "live_mos": null, + "thumbnail_blur_image_url": "", + "image_url_without_letterbox": "", + "diamonds": 0, + "joined_live_thumbnail_image_url": "https://cdn.mirrativ.com/mirrorman-prod/image/live_capture/64c45462a6dc1113cf984a2bebac27d99cbb9b12c736ee3d104dd4c0c4e086db_m.jpeg?1600244617", + "template_comments": [], + "tags": [], + "broadcast_host": "online.mirrativ.com", + "live_user_key": "", + "bcsvr_key": "1abe77db:Cp3UZxJW", + "heartbeated_at": 1600244742, + "shares": { + "twitter": { + "maxlength": 84, + "card": { + "site": "mirrativ.com", + "image_url": "https://cdn.mirrativ.com/mirrorman-prod/image/custom_thumbnail/64c45462a6dc1113cf984a2bebac27d99cbb9b12c736ee3d104dd4c0c4e086db_share.jpeg?1600244617", + "title": "雑談しながらゲームする！", + "description": "🐉そあこ💖爽鱗🐉" + }, + "text": "", + "placeholder": "この配信についてひとこと！" + }, + "others": { + "text": "雑談しながらゲームする！ #プラエデ #ミラティブ #配信 | https://www.mirrativ.com/live/9LtbLw58uSrzig_NTxutIw" + }, + "title": "🐉そあこ💖爽鱗🐉さんの配信を友達にシェアしよう！", + "description": "友達が見に来てくれると配信者さんに伝わるよ！" + }, + "is_private": 0, + "collab_supported": 1, + "sticker_enabled": 0, + "collab_has_vacancy": 0, + "streaming_key": "c4d32ce6edec9a37e7b20cff4b656a95714b5b9ee872964d9e435debe74a9518", + "stamp_num": 0, + "linked_live": { + "owner_badge_image_url": "https://cdn.mirrativ.com/assets/img/continuous_streamer/ic_badge_super_holiday_S.ja.png?v6", + "live_id": "-L6kxuJm_HgJN7aBY5JjYg", + "owner_id": "100895240", + "owner_image_url": "https://cdn.mirrativ.com/mirrorman-prod/image/profile_image/2a6dab73acec21af7d4d08f53f854e4c3a429d0ba73cd3c8f7618f90ff3c4f9b_m.jpeg?1600184061", + "owner_name": "🦔NICO*🦥🥈感謝" + }, + "collab_online_user_num": 0, + "share_image_url": "https://cdn.mirrativ.com/mirrorman-prod/image/custom_thumbnail/64c45462a6dc1113cf984a2bebac27d99cbb9b12c736ee3d104dd4c0c4e086db_share.jpeg?1600244617", + "broadcast_key": "1abe77db:Cp3UZxJW", + "gift_ranking_url": "https://www.mirrativ.com/gift/ranking?live_id=9LtbLw58uSrzig_NTxutIw&obfuscated_user_id=PgIBEgc6jVc", + "collab_mos": null, + "archive_url_hls": "", + "remaining_coins": 0, + "ended_at": 0, + "sticker_category_ids": [], + "online_user_num": 174, + "announcement_url": "", + "anniversary_bot_comment": "", + "is_emomo_wipe_enabled": 0, + "share_url": "https://www.mirrativ.com/live/9LtbLw58uSrzig_NTxutIw", + "status": { + "msg": "", + "ok": 1, + "error": "", + "captcha_url": "", + "error_code": 0, + "message": "" + }, + "orientation": 2, + "app_id": "jp.co.yoozoo.projectyellow", + "app_icon_urls": [ + "https://cdn.mirrativ.com/mirrorman-prod/image/app_icon/999ce961e79bfe6d7fff3db363375a17a112810791a692b8858970efa599ca99.png" + ], + "timeline": [ + { + "app": { + "is_my_app": 0, + "icon_url": "https://cdn.mirrativ.com/mirrorman-prod/image/app_icon/999ce961e79bfe6d7fff3db363375a17a112810791a692b8858970efa599ca99.png", + "store_url": "https://go.onelink.me/app/c8e9133", + "app_id": "jp.co.yoozoo.projectyellow", + "short_title": "プラエデ", + "is_category": 0, + "title": "レッド：プライドオブエデン", + "id": 3139 + }, + "timestamp": 1600242607, + "title": "雑談しながらゲームする！" + } + ], + "enable_clap": 0, + "is_paid_sticker_supported": 0, + "announcement_urls": {}, + "sticker_num": 0, + "max_collab_user_num": 3, + "comment_num": 145, + "owner": { + "share_url": "https://www.mirrativ.com/user/104326294", + "is_able_continuous_stream_holiday": 1, + "profile_image_url": "https://cdn.mirrativ.com/mirrorman-prod/image/profile_image/1046538382ddfccff9b118699ec605cb33cc3c61feb94122f79d5812221e99b5_m.jpeg?1600194531", + "birthday_from": 1615561200, + "is_birthday_editable": 1, + "badges": [ + { + "image_url": "https://cdn.mirrativ.com/assets/img/continuous_streamer/ic_badge_super_holiday_L.ja.png?v6", + "small_image_url": "https://cdn.mirrativ.com/assets/img/continuous_streamer/ic_badge_super_holiday_S.ja.png?v6" + } + ], + "is_new": 0, + "catalog_label_image_url": "https://cdn.mirrativ.com/assets/img/continuous_streamer/ic_status_super_holiday_ja@3x.png?v5", + "birthday": "0314", + "birthday_to": 1615820399, + "name": "🐉そあこ💖爽鱗🐉", + "is_birthday": 0, + "description": "色んなゲームを17:30に定期配信してます♡ギフト当たった人は欲しい時にDM求む", + "birthday_editable_date": "", + "properties": [], + "is_continuous_streamer": 1, + "user_id": "104326294", + "live_request_num": "70", + "onlive": null + }, + "broadcast_port": 1883, + "recommend_sticker_ids": [], + "sticker_display_type": 0, + "archive_comment_enabled": 0, + "streaming_url_edge": "rtmp://edge-210-140-191-149.mirrativ.com:1935/liveedge/c4d32ce6edec9a37e7b20cff4b656a95714b5b9ee872964d9e435debe74a9518", + "collab_enabled": 0, + "image_url": "https://cdn.mirrativ.com/mirrorman-prod/image/thumbnail_image_url/64c45462a6dc1113cf984a2bebac27d99cbb9b12c736ee3d104dd4c0c4e086db_m.jpeg?1600244617", + "orientation_v2": "6" +} + +// curl 'https://www.mirrativ.com/api/gift/ranking?obfuscated_user_id=PgIBEgc6jVc&live_id=O5Ia4iX9c5CeZj7DFtg52Q&type=monthly&cursor=' \ + \ No newline at end of file