test/intimate
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

1.最后一次数据存储结构重构.

Browse Source 
2.数据最接近需求方.
This commit is contained in:
eson
2020-07-20 18:13:54 +08:00
parent 6369387179
commit f0f83a9f00
9 changed files with 296 additions and 140 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
tasks/openrec/openrec_task1/task_openrec.go
View File

@@ -1,7 +1,7 @@
package main
import (
"database/sql"
"encoding/json"
"intimate"
"log"
"os"
@@ -17,8 +17,11 @@ import (
var openrecRanking *OpenrecRanking
// store 源存储实例, 为存储源数据的实现. 表格具体参考sql/intimate_source.sql
var store *intimate.SourceStore = intimate.NewSourceStore(string(intimate.STOpenrec))
// sstore 源存储实例, 为存储源数据的实现. 表格具体参考sql/intimate_source.sql
var sstore *intimate.StoreSource = intimate.NewStoreSource(string(intimate.STOpenrec))
// estore 解析存储连接实例
var estore *intimate.StoreExtractor = intimate.NewStoreExtractor()
func init() {
@@ -69,26 +72,52 @@ func (or *OpenrecRanking) Execute(cxt *hunter.TaskContext) {
content := resp.Content()
if len(content) <= 200 { // 末页退出
finishpoint := time.Now()
log.Println("任务结束休眠, 下次启动时间:", finishpoint.Add(time.Minute*30))
log.Println("任务结束休眠, 下次启动时间:", finishpoint.Add(time.Minute*60))
for time.Now().Sub(finishpoint) < time.Minute*60 {
time.Sleep(time.Second)
if atomic.LoadInt32(&loop) > 0 {
return
}
}
querys := tp.GetQuery()
querys.Set("page", strconv.Itoa(1))
tp.SetQuery(querys)
continue
}
result := gjson.ParseBytes(content)
if result.IsArray() {
for _, User := range result.Array() {
data := &intimate.Source{}
userid := User.Get("channel.id").String()
data.Source = sql.NullString{String: userid, Valid: len(userid) > 0}
data.Url = tp.GetRawURL()
data.TargetType = string(intimate.TTOpenrecUser)
store.Insert(data)
// data := &intimate.Source{}
// data.Source = sql.NullString{String: userid, Valid: len(userid) > 0}
// data.Url = tp.GetRawURL()
// data.TargetType = string(intimate.TTOpenrecUser)
// sstore.Insert(data)
streamer := &intimate.Streamer{}
streamer.UserId = userid
streamer.Platform = string(intimate.Popenrec)
updateUrl := make(map[string]interface{})
supportersUrl := "https://www.openrec.tv/viewapp/api/v6/supporters?identify_id=sumomo_xqx&month=&Uuid=B96EE988-E3A2-4A44-A543-611A8B4BC683&Token=46598c320408bd69ae3c63298f6f4a3a97354175&Random=AZVXNAAXQVMOSVWNDPIQ&page_number=1 -H 'accept: application/json, text/javascript, */*; q=0.01' -H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36' -H 'cookie: uuid=B96EE988-E3A2-4A44-A543-611A8B4BC683;' --compressed"
updateUrl["supporters"] = supportersUrl
updateUrl["user"] = "https://www.openrec.tv/user/" + userid
updateUrl["live"] = "https://www.openrec.tv/live/" + userid
updateUrlBytes, err := json.Marshal(updateUrl)
if err != nil {
estore.UpdateError(streamer, err)
continue
}
streamer.UpdateUrl = updateUrlBytes
estore.InsertStreamer(streamer)
}
}
@@ -102,6 +131,7 @@ func (or *OpenrecRanking) Execute(cxt *hunter.TaskContext) {
page++
querys.Set("page", strconv.Itoa(page))
tp.SetQuery(querys)
time.Sleep(time.Second * 2)
time.Sleep(time.Second * 1)
}
}

7
tasks/openrec/openrec_task1/task_openrec_test.go
View File

@@ -2,6 +2,7 @@ package main
import (
"testing"
"time"
"github.com/tidwall/gjson"
@@ -56,6 +57,12 @@ func TestRanking(t *testing.T) {
ht.Execute()
}
func TestTimeAdd(t *testing.T) {
finishpoint := time.Now()
time.Sleep(time.Second * 2)
t.Error(time.Now().Sub(finishpoint) > time.Second*1)
}
func TestRankingInsert(t *testing.T) {
ht := hunter.NewHunter(openrecRanking)
ht.Execute()

48
tasks/openrec/openrec_task2/task_openrec.go
View File

@@ -20,8 +20,11 @@ import (
var oer *OpenrecExtratorRanking
// store 源存储实例, 为存储源数据的实现. 表格具体参考sql/intimate_source.sql
var store *intimate.SourceStore = intimate.NewSourceStore(string(intimate.STOpenrec))
// sstore 源存储实例, 为存储源数据的实现. 表格具体参考sql/intimate_source.sql
var sstore *intimate.StoreSource = intimate.NewStoreSource(string(intimate.STOpenrec))
// estore 源存储实例, 为存储源数据的实现. 表格具体参考sql/intimate_extractor.sql
var estore *intimate.StoreExtractor = intimate.NewStoreExtractor()
func init() {
oer = &OpenrecExtratorRanking{}
@@ -48,9 +51,9 @@ func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) {
for atomic.LoadInt32(&loop) > 0 {
source, err := store.Pop(string(intimate.TTOpenrecUser))
streamer, err := estore.Pop(string(intimate.Popenrec))
if source == nil || err != nil {
if streamer == nil || err != nil {
if err != lasterr {
log.Println(err, lasterr)
lasterr = err
@@ -59,22 +62,31 @@ func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) {
continue
}
userId := source.Source.String
userUrl := "https://www.openrec.tv/user/" + userId
userId := streamer.UserId
var updateUrl map[string]string
err = json.Unmarshal(streamer.UpdateUrl.([]byte), &updateUrl)
if err != nil {
log.Println(err)
continue
}
// Check Userid
userUrl := updateUrl["user"]
tp := cxt.Session().Get(userUrl)
resp, err := tp.Execute()
source.UpdateTime = sql.NullTime{Time: time.Now(), Valid: true}
streamer.UpdateTime = sql.NullTime{Time: time.Now(), Valid: true}
if err != nil {
log.Println(err)
store.UpdateError(source, err)
estore.UpdateError(streamer, err)
continue
}
cookies := cxt.Session().GetCookies(tp.GetParsedURL())
scurl := "https://www.openrec.tv/viewapp/api/v6/supporters?identify_id=sumomo_xqx&month=&Uuid=B96EE988-E3A2-4A44-A543-611A8B4BC683&Token=46598c320408bd69ae3c63298f6f4a3a97354175&Random=AZVXNAAXQVMOSVWNDPIQ&page_number=1 -H 'accept: application/json, text/javascript, */*; q=0.01' -H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36' -H 'cookie: uuid=B96EE988-E3A2-4A44-A543-611A8B4BC683;' --compressed"
scurl := updateUrl["supporters"]
curl := gcurl.ParseRawCURL(scurl)
supportersSession := curl.CreateSession()
@@ -134,25 +146,31 @@ func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) {
ext["supporters"] = supporters
ext["user"] = string(resp.Content())
tp = cxt.Session().Get("https://www.openrec.tv/live/" + userId)
liveUrl := updateUrl["live"]
tp = cxt.Session().Get(liveUrl)
resp, err = tp.Execute()
if err != nil {
log.Println(err)
store.UpdateError(source, err)
estore.UpdateError(streamer, err)
continue
}
ext["user_live"] = string(resp.Content())
ext["live"] = string(resp.Content())
extJsonBytes, err := json.Marshal(ext)
if err != nil {
log.Println(err)
store.UpdateError(source, err)
estore.UpdateError(streamer, err)
continue
}
source.Operator = int32(intimate.OperatorOK)
streamer.Operator = int32(intimate.OperatorOK)
source := &intimate.Source{}
source.TargetType = string(intimate.TTOpenrecUser)
source.Ext = string(extJsonBytes)
store.Update(source)
sstore.Insert(source)
estore.UpdateOperator(streamer)
}
}