From 4af5430572f5d99b033a062a840b5e0484603480 Mon Sep 17 00:00:00 2001 From: eson Date: Fri, 10 Jul 2020 12:05:33 +0800 Subject: [PATCH 01/14] TODO: extractor openrec --- config.go | 19 +- config.yaml | 3 +- config_test.go | 2 +- extractor/openrec/main.go | 5 + extractor/openrec/openrec_test.go | 37 + source.go | 12 + store.go | 46 +- store_test.go | 4 +- table_list.go | 10 + target_type_list.go | 12 + tasks/openrec/openrec_task1/config.yaml | 1 - tasks/openrec/openrec_task1/task_openrec.go | 6 +- tasks/openrec/openrec_task2/config.yaml | 2 - tasks/openrec/openrec_task2/task_openrec.go | 6 +- testfile/openrec_user.html | 1659 ++++++++++++++++ testfile/openrec_user_live.html | 392 ++++ testfile/openrec_user_supporters.html | 1935 +++++++++++++++++++ 17 files changed, 4117 insertions(+), 34 deletions(-) create mode 100644 extractor/openrec/main.go create mode 100644 extractor/openrec/openrec_test.go create mode 100644 table_list.go create mode 100644 target_type_list.go delete mode 120000 tasks/openrec/openrec_task1/config.yaml delete mode 100644 tasks/openrec/openrec_task2/config.yaml create mode 100755 testfile/openrec_user.html create mode 100755 testfile/openrec_user_live.html create mode 100755 testfile/openrec_user_supporters.html diff --git a/config.go b/config.go index 4b6ddb1..9007be3 100644 --- a/config.go +++ b/config.go @@ -3,6 +3,7 @@ package intimate import ( "errors" "io/ioutil" + "log" "os" "gopkg.in/yaml.v2" @@ -20,20 +21,26 @@ func init() { // Config 配置 type Config struct { Database struct { - URI string `yaml:"uri"` // "user:password@/dbname" + SourceURI string `yaml:"source_uri"` // "user:password@/dbname" + ExtractorURI string `yaml:"extractor_uri"` } `yaml:"database"` } // Load 加载yaml/yml配置 func (conifg *Config) Load() { - configfile := "./config.yaml" - if _, err := os.Stat(configfile); os.IsNotExist(err) { - configfile = "./config.yml" - if _, err := os.Stat(configfile); os.IsNotExist(err) { - panic(errors.New("config.yaml or config.yml is not exists")) + var configfile string + configlist := []string{"./config.yaml", "./config.yml", "../../config.yml", "../../config.yaml"} + for _, configfile = range configlist { + if _, err := os.Stat(configfile); err == nil { + log.Println("find config: ", configfile) + break } } + if len(configfile) <= 4 { + log.Panic(errors.New("can't find config.yaml/config.yml")) + } + f, err := os.Open(configfile) if err != nil { panic(err) diff --git a/config.yaml b/config.yaml index 43d8752..e55d371 100644 --- a/config.yaml +++ b/config.yaml @@ -1,2 +1,3 @@ database: - uri: "root:@tcp(127.0.0.1:4000)/intimate_source" \ No newline at end of file + source_uri: "root:@tcp(127.0.0.1:4000)/intimate_source" + extractor_uri: "root:@tcp(127.0.0.1:4000)/intimate_extractor" \ No newline at end of file diff --git a/config_test.go b/config_test.go index aaffef3..3ef714e 100644 --- a/config_test.go +++ b/config_test.go @@ -6,7 +6,7 @@ func TestConfig(t *testing.T) { config := &Config{} config.Load() - if config.Database.URI != "root:@tcp(127.0.0.1:4000)/intimate_source" { + if config.Database.SourceURI != "root:@tcp(127.0.0.1:4000)/intimate_source" { t.Error("error yaml loaded, ", config) } } diff --git a/extractor/openrec/main.go b/extractor/openrec/main.go new file mode 100644 index 0000000..7905807 --- /dev/null +++ b/extractor/openrec/main.go @@ -0,0 +1,5 @@ +package main + +func main() { + +} diff --git a/extractor/openrec/openrec_test.go b/extractor/openrec/openrec_test.go new file mode 100644 index 0000000..f930b91 --- /dev/null +++ b/extractor/openrec/openrec_test.go @@ -0,0 +1,37 @@ +package main + +import ( + "intimate" + "os" + "testing" + + "github.com/tidwall/gjson" +) + +func TestExtractor(t *testing.T) { + store := intimate.NewSourceStore("source_openrec") + source, err := store.Pop("openrec_user", 100) + if source != nil { + defer store.Restore(source) + } + if err != nil { + t.Error(err) + } + sdata := source.GetExt().([]byte) + + if gjson.ValidBytes(sdata) { + result := gjson.ParseBytes(sdata) + m := result.Map() + for key := range m { + t.Error(key) + f, err := os.OpenFile("./openrec_"+key+".html", os.O_CREATE|os.O_RDWR|os.O_TRUNC, os.ModePerm) + if err != nil { + panic(err) + } + f.WriteString(m[key].String()) + } + } else { + t.Error("data is not json:\n", string(sdata)) + } + +} diff --git a/source.go b/source.go index 2939c22..f4aaf1e 100644 --- a/source.go +++ b/source.go @@ -15,6 +15,18 @@ type Source struct { UpdateTime time.Time // Operator int32 // ErrorMsg sql.NullString // + + lastOperator int32 +} + +// GetLastOperator Get return lastOperator int32 +func (so *Source) GetLastOperator() int32 { + return so.lastOperator +} + +// SetLastOperator Set lastOperator int32 +func (so *Source) SetLastOperator(lastOperator int32) { + so.lastOperator = lastOperator } // GetErrorMsg Get return ErrorMsg sql.NullString diff --git a/store.go b/store.go index cab54e6..268a43d 100644 --- a/store.go +++ b/store.go @@ -24,6 +24,8 @@ type IGetSource interface { type IUpdateSource interface { IGetSource + GetLastOperator() int32 + SetExt(ext interface{}) // SetUpdateTime(ut time.Time) // SetOperator(operator int32) // @@ -42,24 +44,24 @@ const ( OperatorError OperatorFlag = 10000 ) -// Store 储存 -type Store struct { +// SourceStore 储存 +type SourceStore struct { table string db *sql.DB errorCount int errorLimit int } -// NewStore 创建一个存储实例 -func NewStore(table string) *Store { - db, err := sql.Open("mysql", InitConfig.Database.URI) +// NewSourceStore 创建一个存储实例 +func NewSourceStore(table string) *SourceStore { + db, err := sql.Open("mysql", InitConfig.Database.SourceURI) if err != nil { panic(err) } - return &Store{table: table, db: db} + return &SourceStore{table: table, db: db} } -func (store *Store) errorAlarm(err error) { +func (store *SourceStore) errorAlarm(err error) { if err != nil { log.Println("store error: ", err) // 报警. 如果数据插入有问题 @@ -74,20 +76,26 @@ func (store *Store) errorAlarm(err error) { } } -// Insert 储存数据 -func (store *Store) Insert(isource IGetSource) { +// Insert 插入数据 +func (store *SourceStore) Insert(isource IGetSource) { _, err := store.db.Exec("insert into `source_openrec`(url, target_type, source, ext, operator, error_msg) values(?,?,?,?,?,?)", isource.GetUrl(), isource.GetTargetType(), isource.GetSource(), isource.GetExt(), isource.GetOperator(), isource.GetErrorMsg()) store.errorAlarm(err) } -// Update 储存数据 -func (store *Store) Update(isource IUpdateSource) { +// Update 更新数据 +func (store *SourceStore) Update(isource IUpdateSource) { _, err := store.db.Exec("update "+store.table+" set ext = ?, operator = ?, error_msg = ? where uid = ?", isource.GetExt(), isource.GetOperator(), isource.GetErrorMsg(), isource.GetUid()) store.errorAlarm(err) } -// Pop 储存数据 -func (store *Store) Pop(targetType string, operators ...int32) (IUpdateSource, error) { +// Restore 恢复Operator数据状态 +func (store *SourceStore) Restore(isource IUpdateSource) { + _, err := store.db.Exec("update "+store.table+" set operator = ? where uid = ?", isource.GetLastOperator(), isource.GetUid()) + store.errorAlarm(err) +} + +// Pop 弹出一条未处理的数据 +func (store *SourceStore) Pop(targetType string, operators ...int32) (IUpdateSource, error) { tx, err := store.db.Begin() if err != nil { @@ -122,8 +130,11 @@ func (store *Store) Pop(targetType string, operators ...int32) (IUpdateSource, e if row != nil { s := &Source{} + // uid, url, target_type, source, ext, operator err = row.Scan(&s.Uid, &s.Url, &s.TargetType, &s.Source, &s.Ext, &s.Operator) + s.SetLastOperator(s.Operator) + if err != nil { log.Println(err, targetType) _, err = tx.Exec("update "+store.table+" set error_msg = ?, operator = ? where uid = ?", OperatorError, s.Uid) @@ -138,3 +149,12 @@ func (store *Store) Pop(targetType string, operators ...int32) (IUpdateSource, e return nil, errors.New("TaskQueue is nil") } + +// NewExtractorStore 创建一个存储实例 +func NewExtractorStore(table string) *SourceStore { + db, err := sql.Open("mysql", InitConfig.Database.ExtractorURI) + if err != nil { + panic(err) + } + return &SourceStore{table: table, db: db} +} diff --git a/store_test.go b/store_test.go index 5ec7e00..e5264ce 100644 --- a/store_test.go +++ b/store_test.go @@ -26,8 +26,8 @@ func TestStoreInsertCase1(t *testing.T) { } func TestStorePopCase1(t *testing.T) { - store := NewStore("source_openrec") - source, err := store.Pop("openrec_ranking") + store := NewSourceStore("source_openrec") + source, err := store.Pop(string(TTOpenrecRanking)) if err != nil { t.Error(err) } diff --git a/table_list.go b/table_list.go new file mode 100644 index 0000000..2a9fa5b --- /dev/null +++ b/table_list.go @@ -0,0 +1,10 @@ +package intimate + +// SourceTable 源的table列表 +type SourceTable string + +const ( + // STOpenrec openrec源table名称 + STOpenrec SourceTable = "source_openrec" +) + diff --git a/target_type_list.go b/target_type_list.go new file mode 100644 index 0000000..fcd013c --- /dev/null +++ b/target_type_list.go @@ -0,0 +1,12 @@ +package intimate + +// TargetType 源的 目标类型 列表 +type TargetType string + +const ( + // TTOpenrecRanking openrec源TargetType名称 + TTOpenrecRanking TargetType = "openrec_ranking" + + // TTOpenrecUser openrec源TargetType名称 + TTOpenrecUser TargetType = "openrec_ranking" +) diff --git a/tasks/openrec/openrec_task1/config.yaml b/tasks/openrec/openrec_task1/config.yaml deleted file mode 120000 index e416226..0000000 --- a/tasks/openrec/openrec_task1/config.yaml +++ /dev/null @@ -1 +0,0 @@ -../../../config.yaml \ No newline at end of file diff --git a/tasks/openrec/openrec_task1/task_openrec.go b/tasks/openrec/openrec_task1/task_openrec.go index 36cffba..caf8c81 100644 --- a/tasks/openrec/openrec_task1/task_openrec.go +++ b/tasks/openrec/openrec_task1/task_openrec.go @@ -11,12 +11,10 @@ import ( "github.com/tidwall/gjson" ) -var targetTypeRanking = "openrec_ranking" -var targetTypeUser = "openrec_user" var openrecRanking *OpenrecRanking // store 源存储实例, 为存储源数据的实现. 表格具体参考sql/intimate_source.sql -var store *intimate.Store = intimate.NewStore("source_openrec") +var store *intimate.SourceStore = intimate.NewSourceStore(string(intimate.STOpenrec)) func init() { @@ -67,7 +65,7 @@ func (or *OpenrecRanking) Execute(cxt *hunter.TaskContext) { data.SetSource(sql.NullString{String: userid, Valid: len(userid) > 0}) data.SetUrl(wf.GetRawURL()) - data.SetTargetType(targetTypeUser) + data.SetTargetType(string(intimate.TTOpenrecUser)) store.Insert(data) } } diff --git a/tasks/openrec/openrec_task2/config.yaml b/tasks/openrec/openrec_task2/config.yaml deleted file mode 100644 index 43d8752..0000000 --- a/tasks/openrec/openrec_task2/config.yaml +++ /dev/null @@ -1,2 +0,0 @@ -database: - uri: "root:@tcp(127.0.0.1:4000)/intimate_source" \ No newline at end of file diff --git a/tasks/openrec/openrec_task2/task_openrec.go b/tasks/openrec/openrec_task2/task_openrec.go index b74f62b..54931be 100644 --- a/tasks/openrec/openrec_task2/task_openrec.go +++ b/tasks/openrec/openrec_task2/task_openrec.go @@ -10,12 +10,10 @@ import ( "github.com/474420502/hunter" ) -var targetTypeUser = "openrec_user" -var targetTypeRanking = "openrec_ranking" var oer *OpenrecExtratorRanking // store 源存储实例, 为存储源数据的实现. 表格具体参考sql/intimate_source.sql -var store *intimate.Store = intimate.NewStore("source_openrec") +var store *intimate.SourceStore = intimate.NewSourceStore(string(intimate.STOpenrec)) func init() { oer = &OpenrecExtratorRanking{} @@ -31,7 +29,7 @@ func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) { for { - source, err := store.Pop(targetTypeUser) + source, err := store.Pop(string(intimate.TTOpenrecUser)) if err != nil { log.Println(err) return diff --git a/testfile/openrec_user.html b/testfile/openrec_user.html new file mode 100755 index 0000000..6579f80 --- /dev/null +++ b/testfile/openrec_user.html @@ -0,0 +1,1659 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 手越ちゃんねる | OPENREC.tv + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + , + + + + + + + + + + + + + + + + + + + + + + + < + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ +
+ + + + + + + +
+
+ +
+ +
+ + + +
+
+
+ + +
+
+
+
+
+
+
+
Trends
+
+

ピューロ

+
+
+

APEX LEGENDS

+
+
+

FORTNITE

+
+
+

VALORANT

+
+
+

あつまれ どうぶつの森

+
+
+
+
History
+
+
+
+
関連ゲームタイトル
+
+
    +
+
+
+
+
関連チャンネル
+
+
    +
+
+
+
+
関連キーワード
+
+
    +
+
+
+
+ +
+
+
+
+
+ +
+ +
+ + + + + Upgrade to
Premium Account! + +
+ +
+ + + + ダークモード +
+ + +
+
+ +
+
+ + + + + +
+
+ +
+
+ + + + + +
+ +
+ +
+
+

+

+ + +

+
+
+ +
+ +
    +
  • +
    + + + + ダークモード +
    + + +
    +
    +
    • + + +
+
+ + + +
+

Important Notice +

+
+
    +
+ See more +
+ + + +

Notice +

+
+
    +
+
+
+ + + + +
+ + +
+ +
+ +
+
+
    +
  • + + Login +
    • +
  • +
    + + + + +
    +
    • +
  • +
    + + +
    +
    • +
  • +
    + Received on agreement to the Terms of Use,
    Privacy Policy of OPENREC, then you proceed please .
    +
    • + +
  • +
    +
    + Upgrade to Premium Membership +
    + + + There are a bunch of perks that will allow you to enjoy OPENREC.tv even more.
    +
    +
    • +
+ +
+
+ +
+
+ + +
+
+ + +
+
+
+ +
+ +
+
+ + +
+
+
    +
  • + Provisional registration completion +
    • +
  • + We've sent a verification e-mail to the registered e-mail address. Verify your registration by confirming the message.
    • +
  • + If you do not receive the verification email, please check the following .
    • +
+
    +
  • Or there was no error in - your input our e-mail address
    • +
  • Or e-mail from the " @ openrec.jp " domain has not been classified as junk folder
    • +
  • Or e-mail from the " @ openrec.jp " domain is not being blocked by domain rejection setting
+
+ + +
+ +
+
+ + +
+
+
    +
  • + Reregistration your password +
    • +
  • + Enter the email address you've registered.
    +We'll send you an email to reset the password.
    • +
  • + +
    • +
  • +
  • + +
    • +
+ +
+
+ + +
+
+
+
+ Reregistration your password
+
+ We've sent you an email to reset the password.
+
+ +
+ +
+
+ + +
+
+
    +
  • + Reregistration your password +
    • +
  • + Highly secure password is a combination of alphanumeric characters and symbols .
    • +
  • + +
    • +
  • + +
    • +
  • +

    + Reregistration your password

    +
    • +
+ +
+
+ + +
+
+
    +
  • + Report as a Inappropriate movie +
    • +
  • + Types of problem +
    • +
  • + +
    • +
  • + Details of problem +
    • +
  • + +
    • +
  • + We will check if reported contents is violating terms of service
    + To use this function immoderately is also violating terms of service.
    • +
  • +

    + Send

    +
    • +
+ +
+
+ + +
+
+
    +
  • + Report as a spam +
    • +
  • + Details of problem +
    • +
  • + +
    • +
  • + Details of problem +
    • +
  • + +
    • +
  • + We will check if reported contents is violating terms of service
    + To use this function immoderately is also violating terms of service.
    • +
  • +

    + Send

    +
    • +
+ +
+
+ + +
+
+
    +
  • + Report as a Inappropriate movie +
    • +
  • + Thank you for reporting
    + We will check if reported contents is violating terms of service
    +
    • +
+ +
+
+ + +
+
+
    +
  • + Report as a spam +
    • +
  • + Thank you for reporting
    + We will check if reported contents is violating terms of service
    +
    • +
+ +
+
+ +
+
+
    +
  • + For playing video +
    • +
  • + Playing video, it is required to install Flash Player. If you can't play video please download the latest version of Flash Player from here.
    • +
+ +
+
+ + +
+
+
+
+ Your request is pending now.
Please wait for the result.
+
+ Your request got denied.
+
+ You can not apply with this account.
+
+ +
+
+ + + + +
+
+
+ + + +
+
+
    +
  • + To watch a previous live broadcast +
    • +
  • +
    You can get various privileges by upgrading to premium account
    +
    + Chat/comment stamp +
    +
    + Unlimited viewing of archived videos +
    +
    + Special Premium Member only broadcasts +
    +
    • +
  • + Register in only 3 steps!Easy ragestration.¥550/month
    • +
  • + Become premium member +
    • +
+ +
+
+ + +
+
+
    +
  • + 超低遅延モードで視聴するためには、
    プレミアム会員への入会が必要です。     +
    • +
  • +
    You can get various privileges by upgrading to premium account
    +
    + Chat/comment stamp +
    +
    + Unlimited viewing of archived videos +
    +
    + Special Premium Member only broadcasts +
    +
    • +
  • + Register in only 3 steps!Easy ragestration.¥550/month
    • +
  • + Become premium member +
    • +
+ +
+
+ + +
+
+
    +
  • + You can set 200 forbidden words
    if you upgrade to premium account     +
    • +
  • +
    You can get various privileges by upgrading to premium account
    +
    + Chat/comment stamp +
    +
    + Unlimited viewing of archived videos +
    +
    + Special Premium Member only broadcasts +
    +
    • +
  • + Register in only 3 steps!Easy ragestration.¥550/month
    • +
  • + Become premium member +
    • +
+ +
+
+ + +
+
+
    +
  • + Premium user can pop out a chat. +
    • +
  • +
    You can get various privileges by upgrading to premium account
    +
    + Chat/comment stamp +
    +
    + Unlimited viewing of archived videos +
    +
    + Special Premium Member only broadcasts +
    +
    • +
  • + Register in only 3 steps!Easy ragestration.¥550/month
    • +
  • + Become premium member +
    • +
+ +
+
+ +
+
+
+
+ This video has been finished
+
+ +
+ +
+
+ + +
+
+
+
+ Do you want to remove it from the blacklist?
+
+ + +
+ +
+
+ + +
+
+
+
+ Are you sure you want to add to the black list?
+
+ + +
+ +
+
+ + +
+
+
+
+ Do you want to repoart as a spam user?
+
+ + +
+ +
+
+
+
+
+
+
+
+
+
+
+
+
+ +
+
+
+
+
+
+ +
+ +
+
+

Videos

+

5

+
+
+

Views

+

732,990

+
+
+

Followers

+

64,978

+
+
+

Supporter

+

281

+
+
+ + +
+ +
+
+
+
+
+
+
+ 手越ちゃんねる
+
+
+
+
+
+
+ +
+ +
+
+
+

Channels

 + +

+ Capture

+ +

Follow

 +

Follower

 +

Supporter

 +

Your team

 +

Playlist

 +
+
+
+ +
    + + + +
  • +
    +

    Videos

    +
      +
    • +
    • +
    +
    + Filter ▲Filter ▼ +
    +
    +
    +
      +
    • +

      Registration date

      +
      +
        +
      • 〜
        • +
      • Not selected
        • +
      • This month
        • +
      • Last month
        • +
      • More than 3 month ago
        • +
      • More than 6 month ago
        • +
      • More than 1 years
        • +
      +
      +
      • +
    • +

      Type

      +
        +
      • Not selected
        • +
      • Past live
        • +
      • Videos
        • +
      + +
      • +
    • +

      Sort

      +
        +
      • Uploaded date in ascending
        • +
      • Uploaded date in descending
        • +
      • Views
        • +
      + + +
      • +
    • +

      Games

      +
        +
      • Not selected
        • +
      • +
      + +
      • +
    • +

      Category

      +
        +
      • Not selected
        • +
      • +
      + +
      • +
    +
    + +
      + +
      There are no results. Please set another filter.
      + +
        +

        • +
      • There are no videos.
        • +
      +
      • + + + +
    + + + +
    +
    +
    +
    + +
    +
    +
    +
    + Are you sure to remove from the team members
    +
    + + +
    + +
    +
    +
    +
    +
    +
    + Are you sure to add to the team members
    +
    + + +
    + +
    +
    + +
    +
    +
    + + diff --git a/testfile/openrec_user_live.html b/testfile/openrec_user_live.html new file mode 100755 index 0000000..8fd7707 --- /dev/null +++ b/testfile/openrec_user_live.html @@ -0,0 +1,392 @@ + + + + + 自宅からゲーム＆雑談 | OPENREC.tv (オープンレック) + + + + + + + + + + + + + + + + + + + + + + + + +

    @

    ・

    ・

    + + + + + + + + + + + + + + + +
    + + +
    + + + + + + + +
    + +
    + + + \ No newline at end of file diff --git a/testfile/openrec_user_supporters.html b/testfile/openrec_user_supporters.html new file mode 100755 index 0000000..19d322e --- /dev/null +++ b/testfile/openrec_user_supporters.html @@ -0,0 +1,1935 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 手越ちゃんねる | OPENREC.tv + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + , + + + + + + + + + + + + + + + + + + + + + + + < + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    +
    + +
    + + + + + + + +
    +
    + +
    + +
    + + + +
    +
    +
    + + +
    +
    +
    +
    +
    +
    +
    +
    Trends
    +
    +

    ピューロ

    +
    +
    +

    APEX LEGENDS

    +
    +
    +

    FORTNITE

    +
    +
    +

    VALORANT

    +
    +
    +

    あつまれ どうぶつの森

    +
    +
    +
    +
    History
    +
    +
    +
    +
    関連ゲームタイトル
    +
    +
      +
    +
    +
    +
    +
    関連チャンネル
    +
    +
      +
    +
    +
    +
    +
    関連キーワード
    +
    +
      +
    +
    +
    +
    + +
    +
    +
    +
    +
    + +
    + +
    + + + + + Level up to
    Premium Account!     +     +
    + +
    + + + + ダークモード +
    + + +
    +
    + +
    +
    + + + + + +
    +
    + +
    +
    + + + + + +
    + +
    + +
    +
    +

    +

    + + +

    +
    +
    + +
    + +
      +
    • +
      + + + + ダークモード +
      + + +
      +
      +
      • + + +
    +
    + + + +
    +

    Important Notice +

    +
    +
      +
    + See more +
    + + + +

    Notice +

    +
    +
      +
    +
    +
    + + + + +
    + + +
    + +
    + +
    +
    +
      +
    • + + Login +
      • +
    • +
      + + + + +
      +
      • +
    • +
      + + +
      +
      • +
    • +
      + Received on agreement to the Terms of Use,
      Privacy Policy of OPENREC, then you proceed please .
      +
      • + +
    • +
      +
      + Upgrade to Premium Membership +
      + + + There are a bunch of perks that will allow you to enjoy OPENREC.tv even more.
      +
      +
      • +
    + +
    +
    + +
    +
    + + +
    +
    + + +
    +
    +
    + +
    + +
    +
    + + +
    +
    +
      +
    • + Provisional registration completion +
      • +
    • + We've sent a verification e-mail to the registered e-mail address. Verify your registration by confirming the message.
      • +
    • + If you do not receive the verification email, please check the following .
      • +
    +
      +
    • Or there was no error in - your input our e-mail address
      • +
    • Or e-mail from the " @ openrec.jp " domain has not been classified as junk folder
      • +
    • Or e-mail from the " @ openrec.jp " domain is not being blocked by domain rejection setting
    +
    + + +
    + +
    +
    + + +
    +
    +
      +
    • + Reregistration your password +
      • +
    • + Enter the email address you've registered.
      +We'll send you an email to reset the password.
      • +
    • + +
      • +
    • +
    • + +
      • +
    + +
    +
    + + +
    +
    +
    +
    + Reregistration your password
    +
    + We've sent you an email to reset the password.
    +
    + +
    + +
    +
    + + +
    +
    +
      +
    • + Reregistration your password +
      • +
    • + Highly secure password is a combination of alphanumeric characters and symbols .
      • +
    • + +
      • +
    • + +
      • +
    • +

      + Reregistration your password

      +
      • +
    + +
    +
    + + +
    +
    +
      +
    • + Report as a Inappropriate movie +
      • +
    • + Types of problem +
      • +
    • + +
      • +
    • + Details of problem +
      • +
    • + +
      • +
    • + We will check if reported contents is violating terms of service
      + To use this function immoderately is also violating terms of service.
      • +
    • +

      + Send

      +
      • +
    + +
    +
    + + +
    +
    +
      +
    • + Report as a spam +
      • +
    • + Details of problem +
      • +
    • + +
      • +
    • + Details of problem +
      • +
    • + +
      • +
    • + We will check if reported contents is violating terms of service
      + To use this function immoderately is also violating terms of service.
      • +
    • +

      + Send

      +
      • +
    + +
    +
    + + +
    +
    +
      +
    • + Report as a Inappropriate movie +
      • +
    • + Thank you for reporting
      + We will check if reported contents is violating terms of service
      +
      • +
    + +
    +
    + + +
    +
    +
      +
    • + Report as a spam +
      • +
    • + Thank you for reporting
      + We will check if reported contents is violating terms of service
      +
      • +
    + +
    +
    + +
    +
    +
      +
    • + For playing video +
      • +
    • + Playing video, it is required to install Flash Player. If you can't play video please download the latest version of Flash Player from here.
      • +
    + +
    +
    + + +
    +
    +
    +
    + Your request is pending now.
    Please wait for the result.
    +
    + Your request got denied.
    +
    + You can not apply with this account.
    +
    + +
    +
    + + + + +
    +
    +
    + + + +
    +
    +
      +
    • + To watch a previous live broadcast +
      • +
    • +
      You can get various privileges by upgrading to premium account
      +
      + Chat/comment stamp +
      +
      + Unlimited viewing of archived videos +
      +
      + Special Premium Member only broadcasts +
      +
      • +
    • + Register in only 3 steps!Easy ragestration.¥550/month
      • +
    • + Become premium member +
      • +
    + +
    +
    + + +
    +
    +
      +
    • + 超低遅延モードで視聴するためには、
      プレミアム会員への入会が必要です。       +
      • +
    • +
      You can get various privileges by upgrading to premium account
      +
      + Chat/comment stamp +
      +
      + Unlimited viewing of archived videos +
      +
      + Special Premium Member only broadcasts +
      +
      • +
    • + Register in only 3 steps!Easy ragestration.¥550/month
      • +
    • + Become premium member +
      • +
    + +
    +
    + + +
    +
    +
      +
    • + You can set 200 forbidden words
      if you upgrade to premium account       +
      • +
    • +
      You can get various privileges by upgrading to premium account
      +
      + Chat/comment stamp +
      +
      + Unlimited viewing of archived videos +
      +
      + Special Premium Member only broadcasts +
      +
      • +
    • + Register in only 3 steps!Easy ragestration.¥550/month
      • +
    • + Become premium member +
      • +
    + +
    +
    + + +
    +
    +
      +
    • + Premium user can pop out a chat. +
      • +
    • +
      You can get various privileges by upgrading to premium account
      +
      + Chat/comment stamp +
      +
      + Unlimited viewing of archived videos +
      +
      + Special Premium Member only broadcasts +
      +
      • +
    • + Register in only 3 steps!Easy ragestration.¥550/month
      • +
    • + Become premium member +
      • +
    + +
    +
    + +
    +
    +
    +
    + This video has been finished
    +
    + +
    + +
    +
    + + +
    +
    +
    +
    + Do you want to remove it from the blacklist?
    +
    + + +
    + +
    +
    + + +
    +
    +
    +
    + Are you sure you want to add to the black list?
    +
    + + +
    + +
    +
    + + +
    +
    +
    +
    + Do you want to repoart as a spam user?
    +
    + + +
    + +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    + +
    + +
    +
    +

    Videos

    +

    5

    +
    +
    +

    Views

    +

    732,990

    +
    +
    +

    Followers

    +

    64,978

    +
    +
    +

    Supporter

    +

    281

    +
    +
    + + +
    + +
    +
    + +
    +
    +
    +
    +
    + 手越ちゃんねる
    +
    +
    +
    +
    +
    +
    + +
    + +
    +
    +
    +

    Channels

     + +

    + Capture

    +     +

    Follow

     +

    Follower

     +

    Supporter

     +

    Your team

     +

    Playlist

     +
    +
    +
    + + + + + +
    +
    +
    +
    +
    +
    +

    +
    + +
    +
    +
    +
    + Are you sure to add to the team members
    +
    + + +
    + +
    +
    +
    +
    +
    +
    + Are you sure to remove from the team members
    +
    + + +
    + +
    +
    + +
    +
    +
    + + From d5151f92bfbaaa5eeba05f92d0e623730c65854f Mon Sep 17 00:00:00 2001 From: eson Date: Fri, 10 Jul 2020 16:13:08 +0800 Subject: [PATCH 02/14] =?UTF-8?q?1.extractor=20=E8=A1=A8=E5=AD=97=E6=AE=B5?= =?UTF-8?q?=E7=9A=84=E5=AE=9E=E7=8E=B0.=202.=E5=AD=98=E5=82=A8=E9=80=9A?= =?UTF-8?q?=E7=94=A8=E5=8C=85=E7=9A=84=E5=AE=9E=E7=8E=B0.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- extractor/openrec/main.go | 10 + extractor_field.go | 392 +++++++++++++++++++++++++++++++++++ source.go => source_field.go | 23 ++ sql/intimate_extractor.sql | 12 +- store.go | 103 ++++++--- 5 files changed, 503 insertions(+), 37 deletions(-) create mode 100644 extractor_field.go rename source.go => source_field.go (80%) diff --git a/extractor/openrec/main.go b/extractor/openrec/main.go index 7905807..0639189 100644 --- a/extractor/openrec/main.go +++ b/extractor/openrec/main.go @@ -1,5 +1,15 @@ package main +/* + `uid` varchar(36) NOT NULL, + `platform` varchar(255) NOT NULL, + `anchor_id` varchar(255) NOT NULL, + `anchor_name` varchar(255) NOT NULL, + `live_url` text, + `channel` varchar(128) DEFAULT NULL, + `show_type` varchar(255) DEFAULT NULL, +*/ + func main() { } diff --git a/extractor_field.go b/extractor_field.go new file mode 100644 index 0000000..2d15f03 --- /dev/null +++ b/extractor_field.go @@ -0,0 +1,392 @@ +package intimate + +import ( + "database/sql" + "time" +) + +type ISetAnchorInfo interface { + SetUid(int64) // + SetPlatform(string) // + SetAnchorId(string) // + SetAnchorName(string) // + SetLiveUrl(sql.NullString) // + SetChannel(sql.NullString) // + SetShowType(sql.NullString) // + SetExt(interface{}) // + SetUpdateTime(time.Time) // +} + +type IGetAnchorInfo interface { + GetUid() int64 // + GetPlatform() string // + GetAnchorId() string // + GetAnchorName() string // + GetLiveUrl() sql.NullString // + GetChannel() sql.NullString // + GetShowType() sql.NullString // + GetExt() interface{} // + GetUpdateTime() time.Time // +} + +/* +CREATE TABLE `anchor_info` ( + `uid` bigint AUTO_INCREMENT, + `platform` varchar(255) NOT NULL, + `anchor_id` varchar(255) NOT NULL, + `anchor_name` varchar(255) NOT NULL, + `live_url` text, + `channel` varchar(128) DEFAULT NULL, + `show_type` varchar(255) DEFAULT NULL, + `ext` json DEFAULT NULL, + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`uid`), + UNIQUE KEY `platform_anchor_id_idx` (`platform`, `anchor_id`), + KEY `platform_idx` (`platform`), + KEY `anchor_id_idx` (`anchor_id`), + KEY `anchor_name_idx` (`anchor_name`), + KEY `channel_idx` (`channel`), + KEY `show_type_idx` (`show_type`), + KEY `update_time_idx` (`update_time`) +); +*/ + +type AnchorInfo struct { + Uid int64 // + Platform string // + AnchorId string // + AnchorName string // + LiveUrl sql.NullString // + Channel sql.NullString // + ShowType sql.NullString // + Ext interface{} // + UpdateTime time.Time // +} + +// GetUpdateTime Get return UpdateTime time.Time +func (ai *AnchorInfo) GetUpdateTime() time.Time { + return ai.UpdateTime +} + +// SetUpdateTime Set UpdateTime time.Time +func (ai *AnchorInfo) SetUpdateTime(UpdateTime time.Time) { + ai.UpdateTime = UpdateTime +} + +// GetExt Get return Ext interface{} +func (ai *AnchorInfo) GetExt() interface{} { + return ai.Ext +} + +// SetExt Set Ext interface{} +func (ai *AnchorInfo) SetExt(Ext interface{}) { + ai.Ext = Ext +} + +// GetShowType Get return ShowType sql.NullString +func (ai *AnchorInfo) GetShowType() sql.NullString { + return ai.ShowType +} + +// SetShowType Set ShowType sql.NullString +func (ai *AnchorInfo) SetShowType(ShowType sql.NullString) { + ai.ShowType = ShowType +} + +// GetChannel Get return Channel sql.NullString +func (ai *AnchorInfo) GetChannel() sql.NullString { + return ai.Channel +} + +// SetChannel Set Channel sql.NullString +func (ai *AnchorInfo) SetChannel(Channel sql.NullString) { + ai.Channel = Channel +} + +// GetLiveUrl Get return LiveUrl sql.NullString +func (ai *AnchorInfo) GetLiveUrl() sql.NullString { + return ai.LiveUrl +} + +// SetLiveUrl Set LiveUrl sql.NullString +func (ai *AnchorInfo) SetLiveUrl(LiveUrl sql.NullString) { + ai.LiveUrl = LiveUrl +} + +// GetAnchorName Get return AnchorName string +func (ai *AnchorInfo) GetAnchorName() string { + return ai.AnchorName +} + +// SetAnchorName Set AnchorName string +func (ai *AnchorInfo) SetAnchorName(AnchorName string) { + ai.AnchorName = AnchorName +} + +// GetAnchorId Get return AnchorId string +func (ai *AnchorInfo) GetAnchorId() string { + return ai.AnchorId +} + +// SetAnchorId Set AnchorId string +func (ai *AnchorInfo) SetAnchorId(AnchorId string) { + ai.AnchorId = AnchorId +} + +// GetPlatform Get return Platform string +func (ai *AnchorInfo) GetPlatform() string { + return ai.Platform +} + +// SetPlatform Set Platform string +func (ai *AnchorInfo) SetPlatform(Platform string) { + ai.Platform = Platform +} + +// GetUid Get return Uid int64 +func (ai *AnchorInfo) GetUid() int64 { + return ai.Uid +} + +// SetUid Set Uid int64 +func (ai *AnchorInfo) SetUid(Uid int64) { + ai.Uid = Uid +} + +type IGetCollectLog interface { + GetUid() int64 // + GetPlatform() string // + GetAnchorId() string // + GetIsShowing() int32 // + GetIsError() int32 // + GetFollowers() sql.NullInt32 // + GetViews() sql.NullInt32 // + GetGiver() interface{} // + GetGratuity() sql.NullInt32 // + GetShowTitle() sql.NullString // + GetShowStartTime() sql.NullTime // + GetShowEndTime() sql.NullTime // + GetUpdateTime() time.Time // + GetExt() interface{} // + GetError() sql.NullString // +} + +type ISetCollectLog interface { + SetUid(int64) // + SetPlatform(string) // + SetAnchorId(string) // + SetIsShowing(int32) // + SetIsError(int32) // + SetFollowers(sql.NullInt32) // + SetViews(sql.NullInt32) // + SetGiver(interface{}) // + SetGratuity(sql.NullInt32) // + SetShowTitle(sql.NullString) // + SetShowStartTime(sql.NullTime) // + SetShowEndTime(sql.NullTime) // + SetUpdateTime(time.Time) // + SetExt(interface{}) // + SetError(sql.NullString) // +} + +/* +CREATE TABLE `collect_log` ( + `uid` bigint, + `platform` varchar(255) NOT NULL, + `anchor_id` varchar(255) NOT NULL, + + `is_showing` tinyint(1) DEFAULT NULL, + `is_error` tinyint(1) DEFAULT NULL, + + `followers` int(11) DEFAULT NULL, + `views` int(11) DEFAULT NULL, + `giver` json DEFAULT NULL, + `gratuity` int(11) DEFAULT NULL, + + `show_title` text DEFAULT NULL, + `show_start_time` timestamp NULL DEFAULT NULL, + `show_end_time` timestamp NULL DEFAULT NULL, + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, --时间戳从源数据里获取 + `ext` json DEFAULT NULL, + + `error` text DEFAULT NULL, + + KEY `uid_idx` (`uid`), + KEY `platform_idx` (`platform`), + KEY `anchor_id_idx` (`anchor_id`), + KEY `is_showing_idx` (`is_showing`), + KEY `is_error_idx` (`is_error`), + KEY `followers_idx` (`followers`), + KEY `views_idx` (`views`), + KEY `gratuity_idx` (`gratuity`), + KEY `update_time_idx` (`update_time`) +) +*/ + +type CollectLog struct { + Uid int64 // + Platform string // + AnchorId string // + IsShowing int32 // + IsError int32 // + Followers sql.NullInt32 // + Views sql.NullInt32 // + Giver interface{} // + Gratuity sql.NullInt32 // + ShowTitle sql.NullString // + ShowStartTime sql.NullTime // + ShowEndTime sql.NullTime // + UpdateTime time.Time // + Ext interface{} // + Error sql.NullString // +} + +// GetError Get return Error sql.NullString +func (cl *CollectLog) GetError() sql.NullString { + return cl.Error +} + +// SetError Set Error sql.NullString +func (cl *CollectLog) SetError(Error sql.NullString) { + cl.Error = Error +} + +// GetExt Get return Ext interface{} +func (cl *CollectLog) GetExt() interface{} { + return cl.Ext +} + +// SetExt Set Ext interface{} +func (cl *CollectLog) SetExt(Ext interface{}) { + cl.Ext = Ext +} + +// GetUpdateTime Get return UpdateTime time.Time +func (cl *CollectLog) GetUpdateTime() time.Time { + return cl.UpdateTime +} + +// SetUpdateTime Set UpdateTime time.Time +func (cl *CollectLog) SetUpdateTime(UpdateTime time.Time) { + cl.UpdateTime = UpdateTime +} + +// GetShowEndTime Get return ShowEndTime sql.NullTime +func (cl *CollectLog) GetShowEndTime() sql.NullTime { + return cl.ShowEndTime +} + +// SetShowEndTime Set ShowEndTime sql.NullTime +func (cl *CollectLog) SetShowEndTime(ShowEndTime sql.NullTime) { + cl.ShowEndTime = ShowEndTime +} + +// GetShowStartTime Get return ShowStartTime sql.NullTime +func (cl *CollectLog) GetShowStartTime() sql.NullTime { + return cl.ShowStartTime +} + +// SetShowStartTime Set ShowStartTime sql.NullTime +func (cl *CollectLog) SetShowStartTime(ShowStartTime sql.NullTime) { + cl.ShowStartTime = ShowStartTime +} + +// GetShowTitle Get return ShowTitle sql.NullString +func (cl *CollectLog) GetShowTitle() sql.NullString { + return cl.ShowTitle +} + +// SetShowTitle Set ShowTitle sql.NullString +func (cl *CollectLog) SetShowTitle(ShowTitle sql.NullString) { + cl.ShowTitle = ShowTitle +} + +// GetGratuity Get return Gratuity sql.NullInt32 +func (cl *CollectLog) GetGratuity() sql.NullInt32 { + return cl.Gratuity +} + +// SetGratuity Set Gratuity sql.NullInt32 +func (cl *CollectLog) SetGratuity(Gratuity sql.NullInt32) { + cl.Gratuity = Gratuity +} + +// GetGiver Get return Giver interface{} +func (cl *CollectLog) GetGiver() interface{} { + return cl.Giver +} + +// SetGiver Set Giver interface{} +func (cl *CollectLog) SetGiver(Giver interface{}) { + cl.Giver = Giver +} + +// GetViews Get return Views sql.NullInt32 +func (cl *CollectLog) GetViews() sql.NullInt32 { + return cl.Views +} + +// SetViews Set Views sql.NullInt32 +func (cl *CollectLog) SetViews(Views sql.NullInt32) { + cl.Views = Views +} + +// GetFollowers Get return Followers sql.NullInt32 +func (cl *CollectLog) GetFollowers() sql.NullInt32 { + return cl.Followers +} + +// SetFollowers Set Followers sql.NullInt32 +func (cl *CollectLog) SetFollowers(Followers sql.NullInt32) { + cl.Followers = Followers +} + +// GetIsError Get return IsError int32 +func (cl *CollectLog) GetIsError() int32 { + return cl.IsError +} + +// SetIsError Set IsError int32 +func (cl *CollectLog) SetIsError(IsError int32) { + cl.IsError = IsError +} + +// GetIsShowing Get return IsShowing int32 +func (cl *CollectLog) GetIsShowing() int32 { + return cl.IsShowing +} + +// SetIsShowing Set IsShowing int32 +func (cl *CollectLog) SetIsShowing(IsShowing int32) { + cl.IsShowing = IsShowing +} + +// GetAnchorId Get return AnchorId string +func (cl *CollectLog) GetAnchorId() string { + return cl.AnchorId +} + +// SetAnchorId Set AnchorId string +func (cl *CollectLog) SetAnchorId(AnchorId string) { + cl.AnchorId = AnchorId +} + +// GetPlatform Get return Platform string +func (cl *CollectLog) GetPlatform() string { + return cl.Platform +} + +// SetPlatform Set Platform string +func (cl *CollectLog) SetPlatform(Platform string) { + cl.Platform = Platform +} + +// GetUid Get return Uid int64 +func (cl *CollectLog) GetUid() int64 { + return cl.Uid +} + +// SetUid Set Uid int64 +func (cl *CollectLog) SetUid(Uid int64) { + cl.Uid = Uid +} diff --git a/source.go b/source_field.go similarity index 80% rename from source.go rename to source_field.go index f4aaf1e..8e7c77d 100644 --- a/source.go +++ b/source_field.go @@ -5,6 +5,29 @@ import ( "time" ) +// IGetSource 源接口结构 +type IGetSource interface { + GetUid() int64 // + GetUrl() string // + GetTargetType() string // + GetSource() sql.NullString // + GetExt() interface{} // + GetUpdateTime() time.Time // + GetOperator() int32 // + GetErrorMsg() sql.NullString // +} + +type IUpdateSource interface { + IGetSource + + GetLastOperator() int32 + + SetExt(ext interface{}) // + SetUpdateTime(ut time.Time) // + SetOperator(operator int32) // + SetErrorMsg(emsg sql.NullString) // +} + // Source 的结构体 type Source struct { Uid int64 // diff --git a/sql/intimate_extractor.sql b/sql/intimate_extractor.sql index f13534c..5f5728a 100644 --- a/sql/intimate_extractor.sql +++ b/sql/intimate_extractor.sql @@ -2,15 +2,17 @@ create database if not exists `intimate_extractor`; use intimate_extractor; CREATE TABLE IF NOT EXISTS `anchor_info` ( - `uid` varchar(36) NOT NULL, + `uid` bigint AUTO_INCREMENT, `platform` varchar(255) NOT NULL, `anchor_id` varchar(255) NOT NULL, `anchor_name` varchar(255) NOT NULL, `live_url` text, `channel` varchar(128) DEFAULT NULL, `show_type` varchar(255) DEFAULT NULL, + `ext` json DEFAULT NULL, `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, PRIMARY KEY (`uid`), + UNIQUE KEY `platform_anchor_id_idx` (`platform`, `anchor_id`), KEY `platform_idx` (`platform`), KEY `anchor_id_idx` (`anchor_id`), KEY `anchor_name_idx` (`anchor_name`), @@ -19,13 +21,13 @@ CREATE TABLE IF NOT EXISTS `anchor_info` ( KEY `update_time_idx` (`update_time`) ); -CREATE TABLE IF NOT EXISTS `show_log` ( - `uid` varchar(36) NOT NULL, +CREATE TABLE IF NOT EXISTS `collect_log` ( + `uid` bigint, `platform` varchar(255) NOT NULL, `anchor_id` varchar(255) NOT NULL, - `is_showing` tinyint(1) DEFAULT NULL, - `is_error` tinyint(1) DEFAULT NULL, + `is_showing` tinyint(1) DEFAULT 0, + `is_error` tinyint(1) DEFAULT 0, `followers` int(11) DEFAULT NULL, `views` int(11) DEFAULT NULL, diff --git a/store.go b/store.go index 268a43d..9d15378 100644 --- a/store.go +++ b/store.go @@ -4,34 +4,10 @@ import ( "database/sql" "errors" "log" - "time" _ "github.com/go-sql-driver/mysql" ) -// IGetSource 源接口结构 -type IGetSource interface { - GetUid() int64 // - GetUrl() string // - GetTargetType() string // - GetSource() sql.NullString // - GetExt() interface{} // - GetUpdateTime() time.Time // - GetOperator() int32 // - GetErrorMsg() sql.NullString // -} - -type IUpdateSource interface { - IGetSource - - GetLastOperator() int32 - - SetExt(ext interface{}) // - SetUpdateTime(ut time.Time) // - SetOperator(operator int32) // - SetErrorMsg(emsg sql.NullString) // -} - // OperatorFlag 标志 type OperatorFlag int32 @@ -78,7 +54,7 @@ func (store *SourceStore) errorAlarm(err error) { // Insert 插入数据 func (store *SourceStore) Insert(isource IGetSource) { - _, err := store.db.Exec("insert into `source_openrec`(url, target_type, source, ext, operator, error_msg) values(?,?,?,?,?,?)", isource.GetUrl(), isource.GetTargetType(), isource.GetSource(), isource.GetExt(), isource.GetOperator(), isource.GetErrorMsg()) + _, err := store.db.Exec("insert into "+store.table+"(url, target_type, source, ext, operator, error_msg) values(?,?,?,?,?,?)", isource.GetUrl(), isource.GetTargetType(), isource.GetSource(), isource.GetExt(), isource.GetOperator(), isource.GetErrorMsg()) store.errorAlarm(err) } @@ -150,11 +126,74 @@ func (store *SourceStore) Pop(targetType string, operators ...int32) (IUpdateSou return nil, errors.New("TaskQueue is nil") } -// NewExtractorStore 创建一个存储实例 -func NewExtractorStore(table string) *SourceStore { - db, err := sql.Open("mysql", InitConfig.Database.ExtractorURI) - if err != nil { - panic(err) - } - return &SourceStore{table: table, db: db} +// AnchorTable 主播表名称 +const AnchorTable string = "anchor_info" + +// CollectLogTable 采集日志表 +const CollectLogTable string = "collect_log" + +type ExtractorStore struct { + db *sql.DB + + errorCount int + errorLimit int +} + +func (store *ExtractorStore) errorAlarm(err error) { + if err != nil { + log.Println("store error: ", err) + // 报警. 如果数据插入有问题 + store.errorCount++ + if store.errorCount >= store.errorLimit { + // 数据库频繁操作初问题 报警, 减少没意义的请求 + } + } else { + if store.errorCount > 0 { + store.errorCount-- + } + } +} + +/* + `uid` bigint, + `platform` varchar(255) NOT NULL, + `anchor_id` varchar(255) NOT NULL, + `anchor_name` varchar(255) NOT NULL, + `live_url` text, + `channel` varchar(128) DEFAULT NULL, + `show_type` varchar(255) DEFAULT NULL, +*/ + +// InsertAnchorInfo AnchorInfo表, 插入数据 +func (store *ExtractorStore) InsertAnchorInfo(isource IGetAnchorInfo) { + _, err := store.db.Exec("insert into "+AnchorTable+"(platform, anchor_id, anchor_name, live_url, channel, show_type, ext) values(?,?,?,?,?,?,?) ON DUPLICATE KEY UPDATE", isource.GetPlatform(), isource.GetAnchorId(), isource.GetAnchorName(), isource.GetLiveUrl(), isource.GetChannel(), isource.GetShowType(), isource.GetExt()) + store.errorAlarm(err) +} + +/* + `uid` bigint, + `platform` varchar(255) NOT NULL, + `anchor_id` varchar(255) NOT NULL, + + `is_showing` tinyint(1) DEFAULT NULL, + `is_error` tinyint(1) DEFAULT NULL, + + `followers` int(11) DEFAULT NULL, + `views` int(11) DEFAULT NULL, + `giver` json DEFAULT NULL, + `gratuity` int(11) DEFAULT NULL, + + `show_title` text DEFAULT NULL, + `show_start_time` timestamp NULL DEFAULT NULL, + `show_end_time` timestamp NULL DEFAULT NULL, + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, --时间戳从源数据里获取 + `ext` json DEFAULT NULL, + + `error` text DEFAULT NULL, +*/ + +// InsertCollectLog CollectLog表插入数据 +func (store *ExtractorStore) InsertCollectLog(isource IGetAnchorInfo) { + _, err := store.db.Exec("insert into "+CollectLogTable+"(platform, anchor_id, anchor_name, live_url, channel, show_type, ext) values(?,?,?,?,?,?,?) ON DUPLICATE KEY UPDATE", isource.GetPlatform(), isource.GetAnchorId(), isource.GetAnchorName(), isource.GetLiveUrl(), isource.GetChannel(), isource.GetShowType(), isource.GetExt()) + store.errorAlarm(err) } From d258cc51d3f96e2eb6e6f17f025ed20f56233441 Mon Sep 17 00:00:00 2001 From: eson Date: Fri, 10 Jul 2020 18:31:17 +0800 Subject: [PATCH 03/14] for save. --- extractor/openrec/main.go | 2 +- extractor/openrec/openrec_test.go | 10 ++++------ extractor_field.go | 18 +++++++++--------- sql/intimate_extractor.sql | 2 +- store.go | 18 ++++++++++++++---- 5 files changed, 29 insertions(+), 21 deletions(-) diff --git a/extractor/openrec/main.go b/extractor/openrec/main.go index 0639189..86abea9 100644 --- a/extractor/openrec/main.go +++ b/extractor/openrec/main.go @@ -6,7 +6,7 @@ package main `anchor_id` varchar(255) NOT NULL, `anchor_name` varchar(255) NOT NULL, `live_url` text, - `channel` varchar(128) DEFAULT NULL, + `channel` varchar(128) DEFAULT NULL, // 没有分类 `show_type` varchar(255) DEFAULT NULL, */ diff --git a/extractor/openrec/openrec_test.go b/extractor/openrec/openrec_test.go index f930b91..76585ab 100644 --- a/extractor/openrec/openrec_test.go +++ b/extractor/openrec/openrec_test.go @@ -2,13 +2,13 @@ package main import ( "intimate" - "os" "testing" "github.com/tidwall/gjson" ) func TestExtractor(t *testing.T) { + collect := intimate.NewExtractorStore() store := intimate.NewSourceStore("source_openrec") source, err := store.Pop("openrec_user", 100) if source != nil { @@ -24,11 +24,9 @@ func TestExtractor(t *testing.T) { m := result.Map() for key := range m { t.Error(key) - f, err := os.OpenFile("./openrec_"+key+".html", os.O_CREATE|os.O_RDWR|os.O_TRUNC, os.ModePerm) - if err != nil { - panic(err) - } - f.WriteString(m[key].String()) + ai := &intimate.CollectLog{} + ai.SetAnchorId("123") + collect.InsertCollectLog(ai) } } else { t.Error("data is not json:\n", string(sdata)) diff --git a/extractor_field.go b/extractor_field.go index 2d15f03..9aa9cbd 100644 --- a/extractor_field.go +++ b/extractor_field.go @@ -168,7 +168,7 @@ type IGetCollectLog interface { GetShowEndTime() sql.NullTime // GetUpdateTime() time.Time // GetExt() interface{} // - GetError() sql.NullString // + GetErrorMsg() sql.NullString // } type ISetCollectLog interface { @@ -186,7 +186,7 @@ type ISetCollectLog interface { SetShowEndTime(sql.NullTime) // SetUpdateTime(time.Time) // SetExt(interface{}) // - SetError(sql.NullString) // + SetErrorMsg(sql.NullString) // } /* @@ -238,17 +238,17 @@ type CollectLog struct { ShowEndTime sql.NullTime // UpdateTime time.Time // Ext interface{} // - Error sql.NullString // + ErrorMsg sql.NullString // } -// GetError Get return Error sql.NullString -func (cl *CollectLog) GetError() sql.NullString { - return cl.Error +// GetErrorMsg Get return Error sql.NullString +func (cl *CollectLog) GetErrorMsg() sql.NullString { + return cl.ErrorMsg } -// SetError Set Error sql.NullString -func (cl *CollectLog) SetError(Error sql.NullString) { - cl.Error = Error +// SetErrorMsg Set Error sql.NullString +func (cl *CollectLog) SetErrorMsg(ErrorMsg sql.NullString) { + cl.ErrorMsg = ErrorMsg } // GetExt Get return Ext interface{} diff --git a/sql/intimate_extractor.sql b/sql/intimate_extractor.sql index 5f5728a..8979744 100644 --- a/sql/intimate_extractor.sql +++ b/sql/intimate_extractor.sql @@ -40,7 +40,7 @@ CREATE TABLE IF NOT EXISTS `collect_log` ( `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, `ext` json DEFAULT NULL, - `error` text DEFAULT NULL, + `error_msg` text DEFAULT NULL, KEY `uid_idx` (`uid`), KEY `platform_idx` (`platform`), diff --git a/store.go b/store.go index 9d15378..5947247 100644 --- a/store.go +++ b/store.go @@ -141,7 +141,7 @@ type ExtractorStore struct { func (store *ExtractorStore) errorAlarm(err error) { if err != nil { - log.Println("store error: ", err) + log.Panic("store error: ", err) // 报警. 如果数据插入有问题 store.errorCount++ if store.errorCount >= store.errorLimit { @@ -154,6 +154,14 @@ func (store *ExtractorStore) errorAlarm(err error) { } } +func NewExtractorStore() *ExtractorStore { + db, err := sql.Open("mysql", InitConfig.Database.ExtractorURI) + if err != nil { + panic(err) + } + return &ExtractorStore{db: db} +} + /* `uid` bigint, `platform` varchar(255) NOT NULL, @@ -189,11 +197,13 @@ func (store *ExtractorStore) InsertAnchorInfo(isource IGetAnchorInfo) { `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, --时间戳从源数据里获取 `ext` json DEFAULT NULL, - `error` text DEFAULT NULL, + `error_msg` text DEFAULT NULL, */ // InsertCollectLog CollectLog表插入数据 -func (store *ExtractorStore) InsertCollectLog(isource IGetAnchorInfo) { - _, err := store.db.Exec("insert into "+CollectLogTable+"(platform, anchor_id, anchor_name, live_url, channel, show_type, ext) values(?,?,?,?,?,?,?) ON DUPLICATE KEY UPDATE", isource.GetPlatform(), isource.GetAnchorId(), isource.GetAnchorName(), isource.GetLiveUrl(), isource.GetChannel(), isource.GetShowType(), isource.GetExt()) +func (store *ExtractorStore) InsertCollectLog(isource IGetCollectLog) { + _, err := store.db.Exec("insert into "+CollectLogTable+"(uid, platform, anchor_id, is_showing, is_error, followers, views, giver, gratuity, show_title, show_start_time, show_end_time, update_time, ext, error_msg) values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", + isource.GetUid(), isource.GetPlatform(), isource.GetAnchorId(), isource.GetIsShowing(), isource.GetIsError(), isource.GetFollowers(), isource.GetViews(), isource.GetGiver(), isource.GetGratuity(), isource.GetShowTitle(), isource.GetShowStartTime(), isource.GetShowEndTime(), isource.GetUpdateTime(), isource.GetExt(), isource.GetErrorMsg(), + ) store.errorAlarm(err) } From a305700a69deb987f5e7cb40279def564470d130 Mon Sep 17 00:00:00 2001 From: eson Date: Fri, 10 Jul 2020 18:39:27 +0800 Subject: [PATCH 04/14] fix: config path error --- config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.go b/config.go index 9007be3..eb46132 100644 --- a/config.go +++ b/config.go @@ -29,7 +29,7 @@ type Config struct { // Load 加载yaml/yml配置 func (conifg *Config) Load() { var configfile string - configlist := []string{"./config.yaml", "./config.yml", "../../config.yml", "../../config.yaml"} + configlist := []string{"./config.yaml", "./config.yml", "../../config.yml", "../../config.yaml", "../../../config.yml", "../../../config.yaml"} for _, configfile = range configlist { if _, err := os.Stat(configfile); err == nil { log.Println("find config: ", configfile) From d034647b9378eb67dd7035d36d43ac99af28fbe6 Mon Sep 17 00:00:00 2001 From: eson Date: Mon, 13 Jul 2020 18:10:48 +0800 Subject: [PATCH 05/14] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E7=9B=B4=E5=88=B0curl?= =?UTF-8?q?=20=E6=88=90=E5=8A=9F.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config.go | 2 + extractor/openrec/openrec_test.go | 86 +++++-- go.mod | 2 + platform_list.go | 9 + source_field.go | 13 ++ sql/intimate_source.sql | 2 +- store.go | 10 +- tasks/openrec/openrec_task1/task_openrec.go | 2 +- tasks/openrec/openrec_task2/task_openrec.go | 90 ++++++-- testfile/openrec_supporter.json | 235 ++++++++++++++++++++ 10 files changed, 400 insertions(+), 51 deletions(-) create mode 100644 platform_list.go create mode 100644 testfile/openrec_supporter.json diff --git a/config.go b/config.go index eb46132..0b62879 100644 --- a/config.go +++ b/config.go @@ -16,6 +16,8 @@ func init() { InitConfig = &Config{} InitConfig.Load() // storeOpenrec = NewStore() + + log.SetFlags(log.Lshortfile | log.Ldate) } // Config 配置 diff --git a/extractor/openrec/openrec_test.go b/extractor/openrec/openrec_test.go index 76585ab..b4921f4 100644 --- a/extractor/openrec/openrec_test.go +++ b/extractor/openrec/openrec_test.go @@ -1,35 +1,73 @@ package main import ( - "intimate" + "bytes" + "encoding/gob" + "net/http" "testing" - "github.com/tidwall/gjson" + "github.com/474420502/requests" ) func TestExtractor(t *testing.T) { - collect := intimate.NewExtractorStore() - store := intimate.NewSourceStore("source_openrec") - source, err := store.Pop("openrec_user", 100) - if source != nil { - defer store.Restore(source) - } - if err != nil { - t.Error(err) - } - sdata := source.GetExt().([]byte) - if gjson.ValidBytes(sdata) { - result := gjson.ParseBytes(sdata) - m := result.Map() - for key := range m { - t.Error(key) - ai := &intimate.CollectLog{} - ai.SetAnchorId("123") - collect.InsertCollectLog(ai) - } - } else { - t.Error("data is not json:\n", string(sdata)) - } + ses := requests.NewSession() + wf := ses.Get("https://www.openrec.tv/user/Riowh/supporters") + wf.Execute() + + // t.Error(ses.GetCookies(wf.GetParsedURL())) + + buf := bytes.Buffer{} + encoder := gob.NewEncoder(&buf) + encoder.Encode(ses.GetCookies(wf.GetParsedURL())) + + var cookies []*http.Cookie + decoder := gob.NewDecoder(&buf) + t.Error(decoder.Decode(&cookies)) + t.Error(cookies) + + // collect := intimate.NewExtractorStore() + // store := intimate.NewSourceStore("source_openrec") + // source, err := store.Pop(string(intimate.TTOpenrecRanking), 100) + // if source != nil { + // defer store.Restore(source) + // } + // if err != nil { + // t.Error(err) + // } + // sdata := source.GetExt().([]byte) + + // if gjson.ValidBytes(sdata) { + // result := gjson.ParseBytes(sdata) + // m := result.Map() + + // user := m["user"] + + // ai := &intimate.CollectLog{} + // extractor := hunter.NewExtractor([]byte(user.Str)) + // xp, err := extractor.XPathResult("//p[@class='c-global__user__count__row__right js-userCountFollowers']/text()") + // if err != nil { + // t.Error(err) + // } + // if !xp.NodeIter().Next() { + // t.Error("不存在粉丝数") + // } + + // followers := strings.ReplaceAll(xp.String(), ",", "") + + // followersInt, err := strconv.Atoi(followers) + // if err != nil { + // t.Error(err) + // } + + // ai.SetPlatform(string(intimate.Popenrec)) + // ai.SetFollowers(sql.NullInt32{Int32: int32(followersInt), Valid: true}) + // ai.SetAnchorId(source.GetSource().String) + + // collect.InsertCollectLog(ai) + + // } else { + // t.Error("data is not json:\n", string(sdata)) + // } } diff --git a/go.mod b/go.mod index 00a84d7..5d32e9d 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,9 @@ module intimate go 1.14 require ( + github.com/474420502/gcurl v0.0.4 github.com/474420502/hunter v0.1.2 + github.com/474420502/requests v1.5.1 github.com/go-sql-driver/mysql v1.5.0 github.com/go-yaml/yaml v2.1.0+incompatible // indirect github.com/satori/go.uuid v1.2.0 diff --git a/platform_list.go b/platform_list.go new file mode 100644 index 0000000..d694917 --- /dev/null +++ b/platform_list.go @@ -0,0 +1,9 @@ +package intimate + +// Platform 源的table列表 +type Platform string + +const ( + // Popenrec openrec源table名称 + Popenrec Platform = "openrec" +) diff --git a/source_field.go b/source_field.go index 8e7c77d..433736f 100644 --- a/source_field.go +++ b/source_field.go @@ -11,6 +11,7 @@ type IGetSource interface { GetUrl() string // GetTargetType() string // GetSource() sql.NullString // + GetPassGob() sql.NullString // GetExt() interface{} // GetUpdateTime() time.Time // GetOperator() int32 // @@ -22,6 +23,7 @@ type IUpdateSource interface { GetLastOperator() int32 + SetPassGob(sql.NullString) SetExt(ext interface{}) // SetUpdateTime(ut time.Time) // SetOperator(operator int32) // @@ -34,6 +36,7 @@ type Source struct { Url string // TargetType string // Source sql.NullString // + PassGob sql.NullString // Ext interface{} // UpdateTime time.Time // Operator int32 // @@ -42,6 +45,16 @@ type Source struct { lastOperator int32 } +// GetPassGob Get return PassGob sql.NullString +func (so *Source) GetPassGob() sql.NullString { + return so.PassGob +} + +// SetPassGob Set PassGob sql.NullString +func (so *Source) SetPassGob(PassGob sql.NullString) { + so.PassGob = PassGob +} + // GetLastOperator Get return lastOperator int32 func (so *Source) GetLastOperator() int32 { return so.lastOperator diff --git a/sql/intimate_source.sql b/sql/intimate_source.sql index 8e18af3..fac4d51 100644 --- a/sql/intimate_source.sql +++ b/sql/intimate_source.sql @@ -7,7 +7,7 @@ CREATE TABLE IF NOT EXISTS `source_openrec` ( `target_type` varchar(64) NOT NULL, `source` longtext DEFAULT NULL, `ext` json DEFAULT NULL, - + `pass_gob` blob DEFAULT NULL, `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, `operator` int DEFAULT 0, `error_msg` text DEFAULT NULL, diff --git a/store.go b/store.go index 5947247..bfa0596 100644 --- a/store.go +++ b/store.go @@ -60,10 +60,18 @@ func (store *SourceStore) Insert(isource IGetSource) { // Update 更新数据 func (store *SourceStore) Update(isource IUpdateSource) { - _, err := store.db.Exec("update "+store.table+" set ext = ?, operator = ?, error_msg = ? where uid = ?", isource.GetExt(), isource.GetOperator(), isource.GetErrorMsg(), isource.GetUid()) + _, err := store.db.Exec("update "+store.table+" set ext = ?, pass_gob = ?, operator = ?, error_msg = ? where uid = ?", isource.GetExt(), isource.GetPassGob(), isource.GetOperator(), isource.GetErrorMsg(), isource.GetUid()) store.errorAlarm(err) } +// UpdateError 更新错误数据 +func (store *SourceStore) UpdateError(isource IUpdateSource, err error) { + isource.SetOperator(int32(OperatorError)) + isource.SetErrorMsg(sql.NullString{String: err.Error(), Valid: true}) + _, dberr := store.db.Exec("update "+store.table+" set operator = ?, error_msg = ? where uid = ?", isource.GetOperator(), isource.GetErrorMsg(), isource.GetUid()) + store.errorAlarm(dberr) +} + // Restore 恢复Operator数据状态 func (store *SourceStore) Restore(isource IUpdateSource) { _, err := store.db.Exec("update "+store.table+" set operator = ? where uid = ?", isource.GetLastOperator(), isource.GetUid()) diff --git a/tasks/openrec/openrec_task1/task_openrec.go b/tasks/openrec/openrec_task1/task_openrec.go index caf8c81..1aeee12 100644 --- a/tasks/openrec/openrec_task1/task_openrec.go +++ b/tasks/openrec/openrec_task1/task_openrec.go @@ -76,7 +76,7 @@ func (or *OpenrecRanking) Execute(cxt *hunter.TaskContext) { log.Println(err) return } - + return page++ querys.Set("page", strconv.Itoa(page)) wf.SetQuery(querys) diff --git a/tasks/openrec/openrec_task2/task_openrec.go b/tasks/openrec/openrec_task2/task_openrec.go index 54931be..d8cf2ba 100644 --- a/tasks/openrec/openrec_task2/task_openrec.go +++ b/tasks/openrec/openrec_task2/task_openrec.go @@ -1,12 +1,13 @@ package main import ( - "database/sql" "encoding/json" "intimate" "log" "time" + "github.com/474420502/gcurl" + "github.com/474420502/hunter" ) @@ -30,13 +31,11 @@ func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) { for { source, err := store.Pop(string(intimate.TTOpenrecUser)) - if err != nil { - log.Println(err) - return - } - if source == nil { - return + if source == nil && err != nil { + log.Println(err) + time.Sleep(time.Second * 2) + continue } userSource := &intimate.Source{} @@ -50,32 +49,71 @@ func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) { if err != nil { log.Println(err) - - source.SetOperator(int32(intimate.OperatorError)) - source.SetErrorMsg(sql.NullString{String: err.Error(), Valid: true}) + store.UpdateError(source, err) continue } + cookies := cxt.Session().GetCookies(wf.GetParsedURL()) + + scurl := "curl 'https://www.openrec.tv/viewapp/api/v6/supporters?identify_id=sumomo_xqx&month=&Uuid=B96EE988-E3A2-4A44-A543-611A8B4BC683&Token=46598c320408bd69ae3c63298f6f4a3a97354175&Random=AZVXNAAXQVMOSVWNDPIQ&page_number=1' -H 'accept: application/json, text/javascript, */*; q=0.01' -H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36' -H 'cookie: uuid=B96EE988-E3A2-4A44-A543-611A8B4BC683;' --compressed" + curl := gcurl.ParseRawCURL(scurl) + supportersSession := curl.CreateSession() + supportersSession.SetCookies(wf.GetParsedURL(), cookies) + log.Println(curl.ParsedURL) + workflow := curl.CreateWorkflow(supportersSession) + for { + + supportersQuery := workflow.GetQuery() + + for _, cookie := range cookies { + if cookie.Name == "uuid" { + supportersQuery.Set("Uuid", cookie.Value) + continue + } + + if cookie.Name == "token" { + supportersQuery.Set("Token", cookie.Value) + continue + } + + if cookie.Name == "random" { + supportersQuery.Set("Random", cookie.Value) + continue + } + } + + supportersQuery.Set("identify_id", source.GetSource().String) + + workflow.SetQuery(supportersQuery) + log.Println(workflow.GetRawURL()) + resp, err := workflow.Execute() + if err != nil { + log.Println(err) + } else { + log.Println(string(resp.Content())) + } + return + } + // cookies := cxt.Session().GetCookies(wf.GetParsedURL()) + ext := make(map[string]interface{}) ext["user"] = string(resp.Content()) - wf = cxt.Session().Get(userUrl + "/supporters") - resp, err = wf.Execute() - if err != nil { - log.Println(err) - source.SetOperator(int32(intimate.OperatorError)) - source.SetErrorMsg(sql.NullString{String: err.Error(), Valid: true}) - continue - } - ext["user_supporters"] = string(resp.Content()) + // wf = cxt.Session().Get(userUrl + "/supporters") + // resp, err = wf.Execute() + // if err != nil { + // log.Println(err) + // store.UpdateError(source, err) + // continue + // } + // ext["user_supporters"] = string(resp.Content()) wf = cxt.Session().Get("https://www.openrec.tv/live/" + userid) resp, err = wf.Execute() if err != nil { log.Println(err) - source.SetOperator(int32(intimate.OperatorError)) - source.SetErrorMsg(sql.NullString{String: err.Error(), Valid: true}) + store.UpdateError(source, err) continue } ext["user_live"] = string(resp.Content()) @@ -83,15 +121,19 @@ func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) { extJsonBytes, err := json.Marshal(ext) if err != nil { log.Println(err) - source.SetOperator(int32(intimate.OperatorError)) - source.SetErrorMsg(sql.NullString{String: err.Error(), Valid: true}) + store.UpdateError(source, err) continue } + // buf := bytes.Buffer{} + // encoder := gob.NewEncoder(&buf) + // encoder.Encode(cookies) + + // source.SetPassGob(sql.NullString{String: buf.String(), Valid: true}) source.SetOperator(int32(intimate.OperatorOK)) source.SetExt(string(extJsonBytes)) store.Update(source) - + break } } diff --git a/testfile/openrec_supporter.json b/testfile/openrec_supporter.json new file mode 100644 index 0000000..d5dedbd --- /dev/null +++ b/testfile/openrec_supporter.json @@ -0,0 +1,235 @@ +{ + "status": 0, + "data": { + "items": [ + { + "user_id": 655750535, + "user_name": "\u8471(\u306d\u304e)\u3093\u3061\u3087", + "user_icon": null, + "user_key": "NeginCho", + "user_bg": "https:\/\/www.openrec.tv\/viewapp\/images\/v8\/img_back.png", + "background_file": "https:\/\/www.openrec.tv\/viewapp\/images\/v8\/img_back.png", + "user_introduce": "", + "user_type": 2, + "identify_id": "ebev00Nwej3", + "followed_flg": 0, + "movie_total_views": null, + "onair_flg": 0, + "onair": null, + "update_at": null, + "total_yells": 480, + "yell_image_url": "https:\/\/dqd0jw5gvbchn.cloudfront.net\/yell\/22\/2e6d5b4895ae00b656a700e2d25ef41e9e2600cd.gif", + "supporter_rank": 37 + }, + { + "user_id": 97105260, + "user_name": "\u5f71\u306c\u3044", + "user_icon": "https:\/\/openrec-appdata.s3.amazonaws.com\/user\/971053\/97105260.png?1496661867", + "user_key": "touka1308", + "user_bg": "https:\/\/www.openrec.tv\/viewapp\/images\/v8\/img_back.png", + "background_file": "https:\/\/www.openrec.tv\/viewapp\/images\/v8\/img_back.png", + "user_introduce": "", + "user_type": 2, + "identify_id": "PDZBatTy0sDvB", + "followed_flg": 0, + "movie_total_views": null, + "onair_flg": 0, + "onair": null, + "update_at": null, + "total_yells": 480, + "yell_image_url": "https:\/\/dqd0jw5gvbchn.cloudfront.net\/yell\/22\/2e6d5b4895ae00b656a700e2d25ef41e9e2600cd.gif", + "supporter_rank": 38 + }, + { + "user_id": 45079954, + "user_name": "\u3086\u3063\u3061\u3083", + "user_icon": "https:\/\/openrec-appdata.s3.amazonaws.com\/user\/450800\/45079954.png?1482929097", + "user_key": "yuccha1444", + "user_bg": "https:\/\/www.openrec.tv\/viewapp\/images\/v8\/img_back.png", + "background_file": "https:\/\/www.openrec.tv\/viewapp\/images\/v8\/img_back.png", + "user_introduce": "PUBG\u3000\u30b9\u30d7\u30e9\u30c8\u30a5\u30fc\u30f3\u30e1\u30a4\u30f3\u3067\u3059\u3002\n", + "user_type": 2, + "identify_id": "UQIKagVQXfbRX", + "followed_flg": 0, + "movie_total_views": "466", + "onair_flg": 0, + "onair": null, + "update_at": "2020\/7\/12 14:36:11", + "total_yells": 480, + "yell_image_url": "https:\/\/dqd0jw5gvbchn.cloudfront.net\/yell\/22\/2e6d5b4895ae00b656a700e2d25ef41e9e2600cd.gif", + "supporter_rank": 39 + }, + { + "user_id": 72882108, + "user_name": "\u3042\u308b\u3050\u308c", + "user_icon": "https:\/\/openrec-appdata.s3.amazonaws.com\/user\/728822\/72882108.png?1502904526", + "user_key": "foarsiljw10", + "user_bg": "https:\/\/www.openrec.tv\/viewapp\/images\/v8\/img_back.png", + "background_file": "https:\/\/www.openrec.tv\/viewapp\/images\/v8\/img_back.png", + "user_introduce": "", + "user_type": 2, + "identify_id": "O2d6Htfxc8PvB", + "followed_flg": 0, + "movie_total_views": null, + "onair_flg": 0, + "onair": null, + "update_at": null, + "total_yells": 480, + "yell_image_url": "https:\/\/dqd0jw5gvbchn.cloudfront.net\/yell\/22\/2e6d5b4895ae00b656a700e2d25ef41e9e2600cd.gif", + "supporter_rank": 40 + }, + { + "user_id": 709331194, + "user_name": "\u3044\u3044\u3093\u3059\u304b240\u30a8\u30fc\u30eb\uff01\u3042\u3056\u3059\uff01", + "user_icon": null, + "user_key": "oriton", + "user_bg": "https:\/\/www.openrec.tv\/viewapp\/images\/v8\/img_back.png", + "background_file": "https:\/\/www.openrec.tv\/viewapp\/images\/v8\/img_back.png", + "user_introduce": "", + "user_type": 2, + "identify_id": "b8JQo5PvZAJ", + "followed_flg": 0, + "movie_total_views": null, + "onair_flg": 0, + "onair": null, + "update_at": null, + "total_yells": 480, + "yell_image_url": "https:\/\/dqd0jw5gvbchn.cloudfront.net\/yell\/22\/2e6d5b4895ae00b656a700e2d25ef41e9e2600cd.gif", + "supporter_rank": 41 + }, + { + "user_id": 3756302, + "user_name": "\u3048\u3082\u3042\u3044\u3046", + "user_icon": "https:\/\/openrec-appdata.s3.amazonaws.com\/user\/37564\/3756302.png?1457963374", + "user_key": "nininiku", + "user_bg": "https:\/\/www.openrec.tv\/viewapp\/images\/v8\/img_back.png", + "background_file": "https:\/\/www.openrec.tv\/viewapp\/images\/v8\/img_back.png", + "user_introduce": "", + "user_type": 2, + "identify_id": "etB8pDT5kl2Gs", + "followed_flg": 0, + "movie_total_views": null, + "onair_flg": 0, + "onair": null, + "update_at": null, + "total_yells": 400, + "yell_image_url": "https:\/\/dqd0jw5gvbchn.cloudfront.net\/yell\/22\/2e6d5b4895ae00b656a700e2d25ef41e9e2600cd.gif", + "supporter_rank": 42 + }, + { + "user_id": 102189594, + "user_name": "K", + "user_icon": "https:\/\/openrec-appdata.s3.amazonaws.com\/user\/1021896\/102189594.png?1589000488", + "user_key": "sc__pk", + "user_bg": "https:\/\/www.openrec.tv\/viewapp\/images\/v8\/img_back.png", + "background_file": "https:\/\/www.openrec.tv\/viewapp\/images\/v8\/img_back.png", + "user_introduce": "", + "user_type": 2, + "identify_id": "FejYFKk3wcoGV", + "followed_flg": 0, + "movie_total_views": null, + "onair_flg": 0, + "onair": null, + "update_at": null, + "total_yells": 400, + "yell_image_url": "https:\/\/dqd0jw5gvbchn.cloudfront.net\/yell\/22\/2e6d5b4895ae00b656a700e2d25ef41e9e2600cd.gif", + "supporter_rank": 43 + }, + { + "user_id": 193073369, + "user_name": "minomushi", + "user_icon": "https:\/\/openrec-appdata.s3.amazonaws.com\/user\/1930734\/193073369.png?1519733979", + "user_key": "daitaidaitai", + "user_bg": "https:\/\/www.openrec.tv\/viewapp\/images\/v8\/img_back.png", + "background_file": "https:\/\/www.openrec.tv\/viewapp\/images\/v8\/img_back.png", + "user_introduce": "", + "user_type": 2, + "identify_id": "v_QBD_XBpWoAS", + "followed_flg": 0, + "movie_total_views": null, + "onair_flg": 0, + "onair": null, + "update_at": null, + "total_yells": 400, + "yell_image_url": "https:\/\/dqd0jw5gvbchn.cloudfront.net\/yell\/22\/2e6d5b4895ae00b656a700e2d25ef41e9e2600cd.gif", + "supporter_rank": 44 + }, + { + "user_id": 126454208, + "user_name": "\u7d76\u671b\u304a\u3063\u3071\u3043\u3061\u3083\u3093", + "user_icon": "https:\/\/openrec-appdata.s3.amazonaws.com\/user\/1264543\/126454208.png?1575873318", + "user_key": "chiyongo", + "user_bg": "https:\/\/openrec-appdata.s3.amazonaws.com\/user_background\/1264543\/126454208_cover.png?1557337359", + "background_file": "https:\/\/openrec-appdata.s3.amazonaws.com\/user_background\/1264543\/126454208_cover.png?1557337359", + "user_introduce": "\u56fd\u6307\u5b9a\u306e\u96e3\u75c5\u3068\u6226\u3063\u3066\u3044\u307e\u3059\u3002\n\u3067\u3082\u6c17\u6301\u3061\u7684\u306b\u306f\u6bce\u65e5\u5143\u6c17\uff01\uff01\uff01\n\u6bce\u65e5\u30b2\u30fc\u30e0\u3059\u308b\u6642\u9593\u304c\u8db3\u308a\u306a\u3044\u60b2\u3057\u307f\u3002\nTwitter\u219288tico88", + "user_type": 2, + "identify_id": "qWbsDP_t1WjER", + "followed_flg": 0, + "movie_total_views": null, + "onair_flg": 0, + "onair": null, + "update_at": null, + "total_yells": 400, + "yell_image_url": "https:\/\/dqd0jw5gvbchn.cloudfront.net\/yell\/22\/2e6d5b4895ae00b656a700e2d25ef41e9e2600cd.gif", + "supporter_rank": 45 + }, + { + "user_id": 608946713, + "user_name": "\u91ce\u826f\u30aa\u30af\u30bf\u30f3", + "user_icon": null, + "user_key": "ryryryryryryo", + "user_bg": "https:\/\/www.openrec.tv\/viewapp\/images\/v8\/img_back.png", + "background_file": "https:\/\/www.openrec.tv\/viewapp\/images\/v8\/img_back.png", + "user_introduce": "", + "user_type": 2, + "identify_id": "jAnaeQ11WOg", + "followed_flg": 0, + "movie_total_views": null, + "onair_flg": 0, + "onair": null, + "update_at": null, + "total_yells": 400, + "yell_image_url": "https:\/\/dqd0jw5gvbchn.cloudfront.net\/yell\/22\/2e6d5b4895ae00b656a700e2d25ef41e9e2600cd.gif", + "supporter_rank": 46 + }, + { + "user_id": 34933103, + "user_name": "\u306f\u308b\u3061", + "user_icon": "https:\/\/openrec-appdata.s3.amazonaws.com\/user\/349332\/34933103.png?1580674220", + "user_key": "renaryu07131003", + "user_bg": "https:\/\/openrec-appdata.s3.amazonaws.com\/user_background\/349332\/34933103_cover.png?1580674220", + "background_file": "https:\/\/openrec-appdata.s3.amazonaws.com\/user_background\/349332\/34933103_cover.png?1580674220", + "user_introduce": "\u3053\u3093\u306b\u3061\u306f\u3002", + "user_type": 2, + "identify_id": "idqwub_qCyhxd", + "followed_flg": 0, + "movie_total_views": null, + "onair_flg": 0, + "onair": null, + "update_at": null, + "total_yells": 400, + "yell_image_url": "https:\/\/dqd0jw5gvbchn.cloudfront.net\/yell\/22\/2e6d5b4895ae00b656a700e2d25ef41e9e2600cd.gif", + "supporter_rank": 47 + }, + { + "user_id": 651299052, + "user_name": "\u3084\u3080\u304f\u3093", + "user_icon": "https:\/\/openrec-appdata.s3.amazonaws.com\/user\/6512991\/651299052.png?1581381215", + "user_key": "kangsangsoo", + "user_bg": "https:\/\/www.openrec.tv\/viewapp\/images\/v8\/img_back.png", + "background_file": "https:\/\/www.openrec.tv\/viewapp\/images\/v8\/img_back.png", + "user_introduce": "", + "user_type": 2, + "identify_id": "3XvQmvpxLgL", + "followed_flg": 0, + "movie_total_views": null, + "onair_flg": 0, + "onair": null, + "update_at": null, + "total_yells": 400, + "yell_image_url": "https:\/\/dqd0jw5gvbchn.cloudfront.net\/yell\/22\/2e6d5b4895ae00b656a700e2d25ef41e9e2600cd.gif", + "supporter_rank": 48 + } + ] + } +} \ No newline at end of file From ad02a5b4d900b7ce3f50180862239a5c06af1467 Mon Sep 17 00:00:00 2001 From: eson Date: Mon, 13 Jul 2020 19:11:13 +0800 Subject: [PATCH 06/14] =?UTF-8?q?TODO:=20workflow=E6=94=B9=E8=BF=9B,=20?= =?UTF-8?q?=E5=91=BD=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config.go | 2 +- tasks/openrec/openrec_task2/task_openrec.go | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/config.go b/config.go index 0b62879..1e73967 100644 --- a/config.go +++ b/config.go @@ -17,7 +17,7 @@ func init() { InitConfig.Load() // storeOpenrec = NewStore() - log.SetFlags(log.Lshortfile | log.Ldate) + log.SetFlags(log.Llongfile | log.Ldate) } // Config 配置 diff --git a/tasks/openrec/openrec_task2/task_openrec.go b/tasks/openrec/openrec_task2/task_openrec.go index d8cf2ba..defa483 100644 --- a/tasks/openrec/openrec_task2/task_openrec.go +++ b/tasks/openrec/openrec_task2/task_openrec.go @@ -58,9 +58,10 @@ func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) { scurl := "curl 'https://www.openrec.tv/viewapp/api/v6/supporters?identify_id=sumomo_xqx&month=&Uuid=B96EE988-E3A2-4A44-A543-611A8B4BC683&Token=46598c320408bd69ae3c63298f6f4a3a97354175&Random=AZVXNAAXQVMOSVWNDPIQ&page_number=1' -H 'accept: application/json, text/javascript, */*; q=0.01' -H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36' -H 'cookie: uuid=B96EE988-E3A2-4A44-A543-611A8B4BC683;' --compressed" curl := gcurl.ParseRawCURL(scurl) supportersSession := curl.CreateSession() - supportersSession.SetCookies(wf.GetParsedURL(), cookies) + log.Println(curl.ParsedURL) workflow := curl.CreateWorkflow(supportersSession) + supportersSession.SetCookies(workflow.GetParsedURL(), cookies) for { supportersQuery := workflow.GetQuery() @@ -85,7 +86,9 @@ func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) { supportersQuery.Set("identify_id", source.GetSource().String) workflow.SetQuery(supportersQuery) + // workflow.AddCookies(cookies) log.Println(workflow.GetRawURL()) + log.Println(supportersSession.GetCookies(workflow.GetParsedURL())) resp, err := workflow.Execute() if err != nil { log.Println(err) From 0d8b456f41447d527b1b59a3a5c6ace85cb64570 Mon Sep 17 00:00:00 2001 From: eson Date: Tue, 14 Jul 2020 19:00:34 +0800 Subject: [PATCH 07/14] =?UTF-8?q?TODO:=20=E8=A7=A3=E5=86=B3=E7=BC=96?= =?UTF-8?q?=E7=A0=81=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config.yaml | 4 +- extractor/openrec/openrec_test.go | 238 +- extractor_field.go | 46 +- go.mod | 13 +- go.sum | 59 +- source_field.go | 11 +- sql/intimate_extractor.sql | 6 +- store.go | 6 +- .../openrec_task1/source_openrec_test.go | 4 +- tasks/openrec/openrec_task1/task_openrec.go | 2 +- tasks/openrec/openrec_task2/task_openrec.go | 62 +- testfile/openrec_user_supporters.html | 1935 ----------------- testfile/openrec_user_supporters.json | 7 + 13 files changed, 308 insertions(+), 2085 deletions(-) delete mode 100755 testfile/openrec_user_supporters.html create mode 100755 testfile/openrec_user_supporters.json diff --git a/config.yaml b/config.yaml index e55d371..5a5df4a 100644 --- a/config.yaml +++ b/config.yaml @@ -1,3 +1,3 @@ database: - source_uri: "root:@tcp(127.0.0.1:4000)/intimate_source" - extractor_uri: "root:@tcp(127.0.0.1:4000)/intimate_extractor" \ No newline at end of file + source_uri: "root:@tcp(127.0.0.1:4000)/intimate_source?parseTime=true" + extractor_uri: "root:@tcp(127.0.0.1:4000)/intimate_extractor?parseTime=true" \ No newline at end of file diff --git a/extractor/openrec/openrec_test.go b/extractor/openrec/openrec_test.go index b4921f4..bb73e3c 100644 --- a/extractor/openrec/openrec_test.go +++ b/extractor/openrec/openrec_test.go @@ -1,73 +1,219 @@ package main import ( - "bytes" - "encoding/gob" - "net/http" + "database/sql" + "encoding/json" + "intimate" + "log" + "os" + "regexp" + "strconv" + "strings" "testing" + "github.com/474420502/hunter" "github.com/474420502/requests" + "github.com/tidwall/gjson" ) +func preNUm(data byte) int { + var mask byte = 0x80 + var num int = 0 + //8bit中首个0bit前有多少个1bits + for i := 0; i < 8; i++ { + if (data & mask) == mask { + num++ + mask = mask >> 1 + } else { + break + } + } + return num +} +func isUtf8(data []byte) bool { + i := 0 + for i < len(data) { + if (data[i] & 0x80) == 0x00 { + // 0XXX_XXXX + i++ + continue + } else if num := preNUm(data[i]); num > 2 { + // 110X_XXXX 10XX_XXXX + // 1110_XXXX 10XX_XXXX 10XX_XXXX + // 1111_0XXX 10XX_XXXX 10XX_XXXX 10XX_XXXX + // 1111_10XX 10XX_XXXX 10XX_XXXX 10XX_XXXX 10XX_XXXX + // 1111_110X 10XX_XXXX 10XX_XXXX 10XX_XXXX 10XX_XXXX 10XX_XXXX + // preNUm() 返回首个字节的8个bits中首个0bit前面1bit的个数，该数量也是该字符所使用的字节数 + i++ + for j := 0; j < num-1; j++ { + //判断后面的 num - 1 个字节是不是都是10开头 + if (data[i] & 0xc0) != 0x80 { + return false + } + i++ + } + } else { + //其他情况说明不是utf-8 + return false + } + } + return true +} + +func isGBK(data []byte) bool { + length := len(data) + var i int = 0 + for i < length { + if data[i] <= 0x7f { + //编码0~127,只有一个字节的编码，兼容ASCII码 + i++ + continue + } else { + //大于127的使用双字节编码，落在gbk编码范围内的字符 + if data[i] >= 0x81 && + data[i] <= 0xfe && + data[i+1] >= 0x40 && + data[i+1] <= 0xfe && + data[i+1] != 0xf7 { + i += 2 + continue + } else { + return false + } + } + } + return true +} + func TestExtractor(t *testing.T) { ses := requests.NewSession() - wf := ses.Get("https://www.openrec.tv/user/Riowh/supporters") - wf.Execute() + tp := ses.Get("https://www.openrec.tv/user/Riowh/supporters") + tp.Execute() // t.Error(ses.GetCookies(wf.GetParsedURL())) - buf := bytes.Buffer{} - encoder := gob.NewEncoder(&buf) - encoder.Encode(ses.GetCookies(wf.GetParsedURL())) - - var cookies []*http.Cookie - decoder := gob.NewDecoder(&buf) - t.Error(decoder.Decode(&cookies)) - t.Error(cookies) - - // collect := intimate.NewExtractorStore() - // store := intimate.NewSourceStore("source_openrec") - // source, err := store.Pop(string(intimate.TTOpenrecRanking), 100) + collect := intimate.NewExtractorStore() + store := intimate.NewSourceStore("source_openrec") + source, err := store.Pop(string(intimate.TTOpenrecRanking), 100) // if source != nil { // defer store.Restore(source) // } - // if err != nil { - // t.Error(err) - // } - // sdata := source.GetExt().([]byte) + if err != nil { + t.Error(err) + } + sdata := source.GetExt().([]byte) - // if gjson.ValidBytes(sdata) { - // result := gjson.ParseBytes(sdata) - // m := result.Map() + if gjson.ValidBytes(sdata) { + result := gjson.ParseBytes(sdata) + m := result.Map() - // user := m["user"] + user := m["user"] - // ai := &intimate.CollectLog{} - // extractor := hunter.NewExtractor([]byte(user.Str)) - // xp, err := extractor.XPathResult("//p[@class='c-global__user__count__row__right js-userCountFollowers']/text()") - // if err != nil { - // t.Error(err) - // } - // if !xp.NodeIter().Next() { - // t.Error("不存在粉丝数") - // } + ai := &intimate.CollectLog{} + extractor := hunter.NewExtractor([]byte(user.Str)) + xp, err := extractor.XPathResult("//p[@class='c-global__user__count__row__right js-userCountFollowers']/text()") + if err != nil { + t.Error(err) + } + if !xp.NodeIter().Next() { + t.Error("不存在粉丝数") + } - // followers := strings.ReplaceAll(xp.String(), ",", "") + followers := strings.ReplaceAll(xp.String(), ",", "") - // followersInt, err := strconv.Atoi(followers) - // if err != nil { - // t.Error(err) - // } + followersInt, err := strconv.ParseInt(followers, 10, 64) + if err != nil { + t.Error(err) + } - // ai.SetPlatform(string(intimate.Popenrec)) - // ai.SetFollowers(sql.NullInt32{Int32: int32(followersInt), Valid: true}) - // ai.SetAnchorId(source.GetSource().String) + var anchorName string + xp, err = extractor.XPathResult("//p[@class='c-global__user__profile__list__name__text official-icon--after']/text()") + if xp.NodeIter().Next() { + anchorName = xp.String() + } else { + t.Error(err) + } + t.Error(anchorName) - // collect.InsertCollectLog(ai) + // c-contents + xp, err = extractor.XPathResult("//ul[@class='c-contents']//p[@class='c-thumbnailVideo__footer__liveCount']/text()") + if err != nil { + log.Println(err) + } + if xp.NodeIter().Next() { + views := regexp.MustCompile(`[0-9,]+`).FindString(xp.String()) + views = strings.ReplaceAll(views, ",", "") + viewsint, err := strconv.Atoi(views) + if err != nil { + t.Error(err) + } - // } else { - // t.Error("data is not json:\n", string(sdata)) - // } + ai.SetViews(sql.NullInt64{Int64: int64(viewsint), Valid: true}) + ai.SetIsShowing(1) + } + + var givers []interface{} + var gratuity int64 = 0 + giverjson := m["supporters"] + for _, v := range giverjson.Array() { + giverSource := gjson.Parse(v.String()) + for _, item := range giverSource.Get("data.items").Array() { + givers = append(givers, item.Map()) + gratuity += item.Get("total_yells").Int() + } + } + + giversbytes, err := json.Marshal(givers) + if err != nil { + t.Error(err) + ai.SetErrorMsg(sql.NullString{String: err.Error(), Valid: true}) + } else { + ai.SetGiver(giversbytes) + } + + // MovieToolbar__Views-g5e6ic-13 iDRGyA + livejson := m["user_live"] + + f, err := os.OpenFile("./test.html", os.O_CREATE|os.O_TRUNC|os.O_RDWR, os.ModePerm) + if err != nil { + panic(err) + } + f.WriteString(livejson.String()) + t.Error(livejson) + extractor = hunter.NewExtractor([]byte(livejson.Str)) + xr, err := extractor.XPathResult("//h1[ contains(@class, 'MovieTitle__Title')]") + + if err != nil { + t.Error(err) + } + t.Error(xr) + iter := xr.NodeIter() + if iter.Next() { + t.Error(iter.Node().TextContent()) + ai.SetShowTitle(sql.NullString{String: iter.Node().TextContent(), Valid: true}) + + content, err := extractor.XPathResult("//meta[@itemprop='uploadDate']/@content") + if err != nil { + t.Error(err) + } + + if content.NodeIter().Next() { + t.Error(content.String()) + } + } + t.Error(xr.String(), xr.NodeIter().Next(), xr.String()) + + ai.SetGratuity(sql.NullInt64{Int64: gratuity, Valid: true}) + ai.SetPlatform(string(intimate.Popenrec)) + ai.SetFollowers(sql.NullInt64{Int64: int64(followersInt), Valid: true}) + ai.SetAnchorId(source.GetSource().String) + ai.SetUpdateTime(source.GetUpdateTime()) + + collect.InsertCollectLog(ai) + + } else { + t.Error("data is not json:\n", string(sdata)) + } } diff --git a/extractor_field.go b/extractor_field.go index 9aa9cbd..ad5127e 100644 --- a/extractor_field.go +++ b/extractor_field.go @@ -159,14 +159,14 @@ type IGetCollectLog interface { GetAnchorId() string // GetIsShowing() int32 // GetIsError() int32 // - GetFollowers() sql.NullInt32 // - GetViews() sql.NullInt32 // + GetFollowers() sql.NullInt64 // + GetViews() sql.NullInt64 // GetGiver() interface{} // - GetGratuity() sql.NullInt32 // + GetGratuity() sql.NullInt64 // GetShowTitle() sql.NullString // GetShowStartTime() sql.NullTime // GetShowEndTime() sql.NullTime // - GetUpdateTime() time.Time // + GetUpdateTime() sql.NullTime // GetExt() interface{} // GetErrorMsg() sql.NullString // } @@ -177,14 +177,14 @@ type ISetCollectLog interface { SetAnchorId(string) // SetIsShowing(int32) // SetIsError(int32) // - SetFollowers(sql.NullInt32) // - SetViews(sql.NullInt32) // + SetFollowers(sql.NullInt64) // + SetViews(sql.NullInt64) // SetGiver(interface{}) // - SetGratuity(sql.NullInt32) // + SetGratuity(sql.NullInt64) // SetShowTitle(sql.NullString) // SetShowStartTime(sql.NullTime) // SetShowEndTime(sql.NullTime) // - SetUpdateTime(time.Time) // + SetUpdateTime(sql.NullTime) // SetExt(interface{}) // SetErrorMsg(sql.NullString) // } @@ -229,14 +229,14 @@ type CollectLog struct { AnchorId string // IsShowing int32 // IsError int32 // - Followers sql.NullInt32 // - Views sql.NullInt32 // + Followers sql.NullInt64 // + Views sql.NullInt64 // Giver interface{} // - Gratuity sql.NullInt32 // + Gratuity sql.NullInt64 // ShowTitle sql.NullString // ShowStartTime sql.NullTime // ShowEndTime sql.NullTime // - UpdateTime time.Time // + UpdateTime sql.NullTime // Ext interface{} // ErrorMsg sql.NullString // } @@ -262,12 +262,12 @@ func (cl *CollectLog) SetExt(Ext interface{}) { } // GetUpdateTime Get return UpdateTime time.Time -func (cl *CollectLog) GetUpdateTime() time.Time { +func (cl *CollectLog) GetUpdateTime() sql.NullTime { return cl.UpdateTime } // SetUpdateTime Set UpdateTime time.Time -func (cl *CollectLog) SetUpdateTime(UpdateTime time.Time) { +func (cl *CollectLog) SetUpdateTime(UpdateTime sql.NullTime) { cl.UpdateTime = UpdateTime } @@ -302,12 +302,12 @@ func (cl *CollectLog) SetShowTitle(ShowTitle sql.NullString) { } // GetGratuity Get return Gratuity sql.NullInt32 -func (cl *CollectLog) GetGratuity() sql.NullInt32 { +func (cl *CollectLog) GetGratuity() sql.NullInt64 { return cl.Gratuity } // SetGratuity Set Gratuity sql.NullInt32 -func (cl *CollectLog) SetGratuity(Gratuity sql.NullInt32) { +func (cl *CollectLog) SetGratuity(Gratuity sql.NullInt64) { cl.Gratuity = Gratuity } @@ -321,23 +321,23 @@ func (cl *CollectLog) SetGiver(Giver interface{}) { cl.Giver = Giver } -// GetViews Get return Views sql.NullInt32 -func (cl *CollectLog) GetViews() sql.NullInt32 { +// GetViews Get return Views sql.NullInt64 +func (cl *CollectLog) GetViews() sql.NullInt64 { return cl.Views } -// SetViews Set Views sql.NullInt32 -func (cl *CollectLog) SetViews(Views sql.NullInt32) { +// SetViews Set Views sql.NullInt64 +func (cl *CollectLog) SetViews(Views sql.NullInt64) { cl.Views = Views } -// GetFollowers Get return Followers sql.NullInt32 -func (cl *CollectLog) GetFollowers() sql.NullInt32 { +// GetFollowers Get return Followers sql.NullInt64 +func (cl *CollectLog) GetFollowers() sql.NullInt64 { return cl.Followers } // SetFollowers Set Followers sql.NullInt32 -func (cl *CollectLog) SetFollowers(Followers sql.NullInt32) { +func (cl *CollectLog) SetFollowers(Followers sql.NullInt64) { cl.Followers = Followers } diff --git a/go.mod b/go.mod index 5d32e9d..6575876 100644 --- a/go.mod +++ b/go.mod @@ -3,15 +3,12 @@ module intimate go 1.14 require ( - github.com/474420502/gcurl v0.0.4 - github.com/474420502/hunter v0.1.2 - github.com/474420502/requests v1.5.1 + github.com/474420502/gcurl v0.1.2 + github.com/474420502/hunter v0.3.0 + github.com/474420502/requests v1.6.0 github.com/go-sql-driver/mysql v1.5.0 - github.com/go-yaml/yaml v2.1.0+incompatible // indirect - github.com/satori/go.uuid v1.2.0 github.com/tidwall/gjson v1.6.0 github.com/tidwall/pretty v1.0.1 // indirect - golang.org/x/tools v0.0.0-20200708003708-134513de8882 // indirect - gopkg.in/yaml.v2 v2.2.2 - honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a + golang.org/x/net v0.0.0-20200707034311-ab3426394381 // indirect + gopkg.in/yaml.v2 v2.3.0 ) diff --git a/go.sum b/go.sum index 942a89d..7a07e84 100644 --- a/go.sum +++ b/go.sum @@ -2,21 +2,29 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.41.0/go.mod h1:OauMR7DV8fzvZIl2qg6rkaIhD/vmgk4iwEw/h6ercmg= -github.com/474420502/focus v0.9.0 h1:Y/TjSdSdIgegO78OJixphiPl1wVqhK9AbcXjiDDugo4= -github.com/474420502/focus v0.9.0/go.mod h1:jrDXvK1CnUJ3PCR3ZJVYinbS2Yz5kM8OoAbCLe6AF7Y= -github.com/474420502/gcurl v0.0.4 h1:eR1BNXvQ4T245dotWpjDzAMWch+FTTfScqzsdq93JK0= -github.com/474420502/gcurl v0.0.4/go.mod h1:qtCzAZZbVRIsBt0lNUh2I0qDniU9T3E21aSsVUYo7Hc= -github.com/474420502/hunter v0.1.2 h1:1dY2C9IJvh81+04Xx8OuTPcUw6WNlKCiOaGmJdgX83Y= -github.com/474420502/hunter v0.1.2/go.mod h1:41DpZWSsGBWsFwb/liapbT1uH58Yvl+BpW4SJwLC2Fw= -github.com/474420502/requests v1.5.0/go.mod h1:SLXrQ5dL9c7dkIeKNUCBAjOIt3J9KFCS2RQjWJecNwo= -github.com/474420502/requests v1.5.1 h1:miv6O4RMbZ8I0ZdUTLf/EU5Dmewc/4IL/DmUMwtuv8M= -github.com/474420502/requests v1.5.1/go.mod h1:SLXrQ5dL9c7dkIeKNUCBAjOIt3J9KFCS2RQjWJecNwo= +github.com/474420502/focus v0.12.0 h1:+icbmj7IEOefvTegHt5EpcHt6WFbe2miIrceUJx2Evo= +github.com/474420502/focus v0.12.0/go.mod h1:d0PMjtMxFz1a9HIhwyFPkWa+JF+0LgOrEUfd8iZka6s= +github.com/474420502/gcurl v0.1.0 h1:aKPP27qAnofntTqqVF8rjejHBWVlYWzFEZGdqjBiMgw= +github.com/474420502/gcurl v0.1.0/go.mod h1:hws5q/Ao64bXLLDnldz9VyTQUndTWc/i5DzdEazFfoM= +github.com/474420502/gcurl v0.1.2 h1:ON9Yz3IgAdtDlFlHfkAJ3aIEBDxH0RiViPE5ST5ohKg= +github.com/474420502/gcurl v0.1.2/go.mod h1:hws5q/Ao64bXLLDnldz9VyTQUndTWc/i5DzdEazFfoM= +github.com/474420502/hunter v0.2.0 h1:pH7xIWzas2IuLdCJL/HtcouHcZQq0XIP/FswY7yF+pA= +github.com/474420502/hunter v0.2.0/go.mod h1:c1+92qUtjNzYw6Mzl6Qkb2kMALAXMlYQk3kJdoOqnmY= +github.com/474420502/hunter v0.3.0 h1:0VPi1MInxjHOta3da4v0ALWK0y3/X4/6nUSLFvdbiFU= +github.com/474420502/hunter v0.3.0/go.mod h1:pe4Xr/I+2agvq339vS/OZV+EiHAWtpXQs75rioSW9oA= +github.com/474420502/requests v1.6.0 h1:f4h4j40eT0P5whhg9LdkotD8CaKjtuDu/vz9iSUkCgY= +github.com/474420502/requests v1.6.0/go.mod h1:SLXrQ5dL9c7dkIeKNUCBAjOIt3J9KFCS2RQjWJecNwo= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802 h1:1BDTz0u9nC3//pOCMdNH+CiXJVYJh5UQNCOBG7jbELc= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/BurntSushi/xgbutil v0.0.0-20160919175755-f7c97cef3b4e h1:4ZrkT/RzpnROylmoQL57iVUL57wGKTR5O6KpVnbm2tA= github.com/BurntSushi/xgbutil v0.0.0-20160919175755-f7c97cef3b4e/go.mod h1:uw9h2sd4WWHOPdJ13MQpwK5qYWKYDumDqxWWIknEQ+k= github.com/Pallinder/go-randomdata v1.1.0 h1:gUubB1IEUliFmzjqjhf+bgkg1o6uoFIkRsP3VrhEcx8= github.com/Pallinder/go-randomdata v1.1.0/go.mod h1:yHmJgulpD2Nfrm0cR9tI/+oAgRqCQQixsA8HyRZfV9Y= +github.com/Pallinder/go-randomdata v1.2.0 h1:DZ41wBchNRb/0GfsePLiSwb0PHZmT67XY00lCDlaYPg= +github.com/Pallinder/go-randomdata v1.2.0/go.mod h1:yHmJgulpD2Nfrm0cR9tI/+oAgRqCQQixsA8HyRZfV9Y= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= @@ -24,12 +32,13 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/elazarl/goproxy v0.0.0-20190711103511-473e67f1d7d2 h1:aZtFdDNWY/yH86JPR2WX/PN63635VsE/f/nXNPAbYxY= github.com/elazarl/goproxy v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2 h1:dWB6v3RcOy03t/bUadywsbyrQwCqZeNIEX6M1OtSZOM= github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8= github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= -github.com/go-yaml/yaml v2.1.0+incompatible h1:RYi2hDdss1u4YE7GwixGzWwVo47T8UQwnTLB6vQiq+o= -github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -39,6 +48,7 @@ github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-github/v27 v27.0.4/go.mod h1:/0Gr8pJ55COkmv+S/yPKCczSkUPIM/LnFyubufRNIS0= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= @@ -54,11 +64,11 @@ github.com/lestrrat-go/libxml2 v0.0.0-20200215080510-6483566f52cb h1:qqNmX9V9n4b github.com/lestrrat-go/libxml2 v0.0.0-20200215080510-6483566f52cb/go.mod h1:fy/ZVbgyB83mtricxwSW3zqIRXWOVpKG2PvdUDFeC58= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc= -github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= -github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/tebeka/selenium v0.9.9 h1:cNziB+etNgyH/7KlNI7RMC1ua5aH1+5wUlFQyzeMh+w= github.com/tebeka/selenium v0.9.9/go.mod h1:5Fr8+pUvU6B1OiPfkdCKdXZyr5znvVkxuPd0NOdZCQc= @@ -71,12 +81,10 @@ github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/pretty v1.0.1 h1:WE4RBSZ1x6McVVC8S/Md+Qse8YUv6HRObAx6ke00NY8= github.com/tidwall/pretty v1.0.1/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -87,7 +95,6 @@ golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTk golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -100,8 +107,8 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344 h1:vGXIOMxbNfDTk/aXCmfdLgkrSV+Z2tcbze+pEc3v5W4= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -110,7 +117,6 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -122,6 +128,7 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -136,12 +143,6 @@ golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190624190245-7f2218787638 h1:uIfBkD8gLczr4XDgYpt/qJYds2YJwZRNw4zs7wSnNhk= golang.org/x/tools v0.0.0-20190624190245-7f2218787638/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200708003708-134513de8882 h1:x4Two2lSwHxTqR+eal4lB4ydUnTvmDDpPQeL92ZHDgA= -golang.org/x/tools v0.0.0-20200708003708-134513de8882/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= @@ -159,10 +160,18 @@ google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiq google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/xmlpath.v1 v1.0.0-20140413065638-a146725ea6e7 h1:zibSPXbkfB1Dwl76rJgLa68xcdHu42qmFTe6vAnU4wA= +gopkg.in/xmlpath.v1 v1.0.0-20140413065638-a146725ea6e7/go.mod h1:wo0SW5T6XqIKCCAge330Cd5sm+7VI6v85OrQHIk50KM= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a h1:LJwr7TCTghdatWv40WobzlKXc9c4s8oGa7QKJUtHhWA= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +launchpad.net/gocheck v0.0.0-20140225173054-000000000087 h1:Izowp2XBH6Ya6rv+hqbceQyw/gSGoXfH/UPoTGduL54= +launchpad.net/gocheck v0.0.0-20140225173054-000000000087/go.mod h1:hj7XX3B/0A+80Vse0e+BUHsHMTEhd0O4cpUHr/e/BUM= +launchpad.net/xmlpath v0.0.0-20130614043138-000000000004 h1:B8nNZBUrx8YufDCAJjvO/lVs4GxXMQHyrjwJdJzXMFg= +launchpad.net/xmlpath v0.0.0-20130614043138-000000000004/go.mod h1:vqyExLOM3qBx7mvYRkoxjSCF945s0mbe7YynlKYXtsA= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= diff --git a/source_field.go b/source_field.go index 433736f..b8d601c 100644 --- a/source_field.go +++ b/source_field.go @@ -2,7 +2,6 @@ package intimate import ( "database/sql" - "time" ) // IGetSource 源接口结构 @@ -13,7 +12,7 @@ type IGetSource interface { GetSource() sql.NullString // GetPassGob() sql.NullString // GetExt() interface{} // - GetUpdateTime() time.Time // + GetUpdateTime() sql.NullTime // GetOperator() int32 // GetErrorMsg() sql.NullString // } @@ -25,7 +24,7 @@ type IUpdateSource interface { SetPassGob(sql.NullString) SetExt(ext interface{}) // - SetUpdateTime(ut time.Time) // + SetUpdateTime(ut sql.NullTime) // SetOperator(operator int32) // SetErrorMsg(emsg sql.NullString) // } @@ -38,7 +37,7 @@ type Source struct { Source sql.NullString // PassGob sql.NullString // Ext interface{} // - UpdateTime time.Time // + UpdateTime sql.NullTime // Operator int32 // ErrorMsg sql.NullString // @@ -86,12 +85,12 @@ func (so *Source) SetOperator(Operator int32) { } // GetUpdateTime Get return UpdateTime time.Time -func (so *Source) GetUpdateTime() time.Time { +func (so *Source) GetUpdateTime() sql.NullTime { return so.UpdateTime } // SetUpdateTime Set UpdateTime time.Time -func (so *Source) SetUpdateTime(UpdateTime time.Time) { +func (so *Source) SetUpdateTime(UpdateTime sql.NullTime) { so.UpdateTime = UpdateTime } diff --git a/sql/intimate_extractor.sql b/sql/intimate_extractor.sql index 8979744..cb1d1e5 100644 --- a/sql/intimate_extractor.sql +++ b/sql/intimate_extractor.sql @@ -29,10 +29,10 @@ CREATE TABLE IF NOT EXISTS `collect_log` ( `is_showing` tinyint(1) DEFAULT 0, `is_error` tinyint(1) DEFAULT 0, - `followers` int(11) DEFAULT NULL, - `views` int(11) DEFAULT NULL, + `followers` bigint(11) DEFAULT NULL, + `views` bigint(11) DEFAULT NULL, `giver` json DEFAULT NULL, - `gratuity` int(11) DEFAULT NULL, + `gratuity` bigint(11) DEFAULT NULL, `show_title` text DEFAULT NULL, `show_start_time` timestamp NULL DEFAULT NULL, diff --git a/store.go b/store.go index bfa0596..4810691 100644 --- a/store.go +++ b/store.go @@ -87,7 +87,7 @@ func (store *SourceStore) Pop(targetType string, operators ...int32) (IUpdateSou return nil, err } var args = []interface{}{targetType} - selectSQL := `select uid, url, target_type, source, ext, operator from ` + store.table + ` where target_type = ?` + selectSQL := `select uid, url, target_type, source, ext, operator, update_time from ` + store.table + ` where target_type = ?` if len(operators) == 0 { selectSQL += " and operator = ?" args = append(args, 0) @@ -116,12 +116,12 @@ func (store *SourceStore) Pop(targetType string, operators ...int32) (IUpdateSou s := &Source{} // uid, url, target_type, source, ext, operator - err = row.Scan(&s.Uid, &s.Url, &s.TargetType, &s.Source, &s.Ext, &s.Operator) + err = row.Scan(&s.Uid, &s.Url, &s.TargetType, &s.Source, &s.Ext, &s.Operator, &s.UpdateTime) s.SetLastOperator(s.Operator) if err != nil { log.Println(err, targetType) - _, err = tx.Exec("update "+store.table+" set error_msg = ?, operator = ? where uid = ?", OperatorError, s.Uid) + _, err = tx.Exec("update "+store.table+" set error_msg = ?, operator = ? where uid = ?", err.Error(), OperatorError, s.Uid) if err != nil { log.Println(err) } diff --git a/tasks/openrec/openrec_task1/source_openrec_test.go b/tasks/openrec/openrec_task1/source_openrec_test.go index 8eddfd6..b21c538 100644 --- a/tasks/openrec/openrec_task1/source_openrec_test.go +++ b/tasks/openrec/openrec_task1/source_openrec_test.go @@ -29,8 +29,8 @@ func (or *OpenrecRankingTest) Execute(cxt *hunter.TaskContext) { t.Error("rank is error. result raw is ", result.Raw) } - if cxt.Workflow().GetQuery().Get("page") != "1" { - t.Error("workflow page error") + if cxt.Temporary().GetQuery().Get("page") != "1" { + t.Error("Temporary page error") } // t.Error(string(resp.Content())) } diff --git a/tasks/openrec/openrec_task1/task_openrec.go b/tasks/openrec/openrec_task1/task_openrec.go index 1aeee12..7976b96 100644 --- a/tasks/openrec/openrec_task1/task_openrec.go +++ b/tasks/openrec/openrec_task1/task_openrec.go @@ -50,7 +50,7 @@ func (or *OpenrecRanking) Execute(cxt *hunter.TaskContext) { break } - wf := cxt.Workflow() + wf := cxt.Temporary() content := resp.Content() if len(content) <= 200 { diff --git a/tasks/openrec/openrec_task2/task_openrec.go b/tasks/openrec/openrec_task2/task_openrec.go index defa483..6610541 100644 --- a/tasks/openrec/openrec_task2/task_openrec.go +++ b/tasks/openrec/openrec_task2/task_openrec.go @@ -1,12 +1,15 @@ package main import ( + "database/sql" "encoding/json" "intimate" "log" + "strconv" "time" "github.com/474420502/gcurl" + "github.com/tidwall/gjson" "github.com/474420502/hunter" ) @@ -45,7 +48,7 @@ func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) { wf := cxt.Session().Get(userUrl) resp, err := wf.Execute() - source.SetUpdateTime(time.Now()) + source.SetUpdateTime(sql.NullTime{Time: time.Now(), Valid: true}) if err != nil { log.Println(err) @@ -55,16 +58,16 @@ func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) { cookies := cxt.Session().GetCookies(wf.GetParsedURL()) - scurl := "curl 'https://www.openrec.tv/viewapp/api/v6/supporters?identify_id=sumomo_xqx&month=&Uuid=B96EE988-E3A2-4A44-A543-611A8B4BC683&Token=46598c320408bd69ae3c63298f6f4a3a97354175&Random=AZVXNAAXQVMOSVWNDPIQ&page_number=1' -H 'accept: application/json, text/javascript, */*; q=0.01' -H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36' -H 'cookie: uuid=B96EE988-E3A2-4A44-A543-611A8B4BC683;' --compressed" + scurl := "https://www.openrec.tv/viewapp/api/v6/supporters?identify_id=sumomo_xqx&month=&Uuid=B96EE988-E3A2-4A44-A543-611A8B4BC683&Token=46598c320408bd69ae3c63298f6f4a3a97354175&Random=AZVXNAAXQVMOSVWNDPIQ&page_number=1 -H 'accept: application/json, text/javascript, */*; q=0.01' -H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36' -H 'cookie: uuid=B96EE988-E3A2-4A44-A543-611A8B4BC683;' --compressed" curl := gcurl.ParseRawCURL(scurl) supportersSession := curl.CreateSession() - log.Println(curl.ParsedURL) - workflow := curl.CreateWorkflow(supportersSession) - supportersSession.SetCookies(workflow.GetParsedURL(), cookies) + temporary := curl.CreateTemporary(supportersSession) + supportersSession.SetCookies(temporary.GetParsedURL(), cookies) + var supporters []string for { - supportersQuery := workflow.GetQuery() + supportersQuery := temporary.GetQuery() for _, cookie := range cookies { if cookie.Name == "uuid" { @@ -84,34 +87,37 @@ func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) { } supportersQuery.Set("identify_id", source.GetSource().String) + temporary.SetQuery(supportersQuery) - workflow.SetQuery(supportersQuery) - // workflow.AddCookies(cookies) - log.Println(workflow.GetRawURL()) - log.Println(supportersSession.GetCookies(workflow.GetParsedURL())) - resp, err := workflow.Execute() + resp, err := temporary.Execute() if err != nil { log.Println(err) - } else { - log.Println(string(resp.Content())) } - return - } - // cookies := cxt.Session().GetCookies(wf.GetParsedURL()) + supporterjson := gjson.ParseBytes(resp.Content()) + supporterdata := supporterjson.Get("data") + if supporterdata.Type == gjson.Null { + break + } + supporters = append(supporters, string(resp.Content())) + page := supportersQuery.Get("page_number") + pageint, err := strconv.Atoi(page) + if err != nil { + log.Println(err) + break + } + pageint++ + page = strconv.Itoa(pageint) + supportersQuery.Set("page_number", page) + temporary.SetQuery(supportersQuery) + } + + // cookies := cxt.Session().GetCookies(wf.GetParsedURL()) ext := make(map[string]interface{}) + ext["supporters"] = supporters ext["user"] = string(resp.Content()) - // wf = cxt.Session().Get(userUrl + "/supporters") - // resp, err = wf.Execute() - // if err != nil { - // log.Println(err) - // store.UpdateError(source, err) - // continue - // } - // ext["user_supporters"] = string(resp.Content()) - wf = cxt.Session().Get("https://www.openrec.tv/live/" + userid) resp, err = wf.Execute() if err != nil { @@ -128,15 +134,9 @@ func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) { continue } - // buf := bytes.Buffer{} - // encoder := gob.NewEncoder(&buf) - // encoder.Encode(cookies) - - // source.SetPassGob(sql.NullString{String: buf.String(), Valid: true}) source.SetOperator(int32(intimate.OperatorOK)) source.SetExt(string(extJsonBytes)) store.Update(source) - break } } diff --git a/testfile/openrec_user_supporters.html b/testfile/openrec_user_supporters.html deleted file mode 100755 index 19d322e..0000000 --- a/testfile/openrec_user_supporters.html +++ /dev/null @@ -1,1935 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 手越ちゃんねる | OPENREC.tv - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - , - - - - - - - - - - - - - - - - - - - - - - - < - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    -
    - -
    - - - - - - - -
    -
    - -
    - -
    - - - -
    -
    -
    - - -
    -
    -
    -
    -
    -
    -
    -
    Trends
    -
    -

    ピューロ

    -
    -
    -

    APEX LEGENDS

    -
    -
    -

    FORTNITE

    -
    -
    -

    VALORANT

    -
    -
    -

    あつまれ どうぶつの森

    -
    -
    -
    -
    History
    -
    -
    -
    -
    関連ゲームタイトル
    -
    -
      -
    -
    -
    -
    -
    関連チャンネル
    -
    -
      -
    -
    -
    -
    -
    関連キーワード
    -
    -
      -
    -
    -
    -
    - -
    -
    -
    -
    -
    - -
    - -
    - - - - - Level up to
    Premium Account!     -     -
    - -
    - - - - ダークモード -
    - - -
    -
    - -
    -
    - - - - - -
    -
    - -
    -
    - - - - - -
    - -
    - -
    -
    -

    -

    - - -

    -
    -
    - -
    - -
      -
    • -
      - - - - ダークモード -
      - - -
      -
      -
      • - - -
    -
    - - - -
    -

    Important Notice -

    -
    -
      -
    - See more -
    - - - -

    Notice -

    -
    -
      -
    -
    -
    - - - - -
    - - -
    - -
    - -
    -
    -
      -
    • - - Login -
      • -
    • -
      - - - - -
      -
      • -
    • -
      - - -
      -
      • -
    • -
      - Received on agreement to the Terms of Use,
      Privacy Policy of OPENREC, then you proceed please .
      -
      • - -
    • -
      -
      - Upgrade to Premium Membership -
      - - - There are a bunch of perks that will allow you to enjoy OPENREC.tv even more.
      -
      -
      • -
    - -
    -
    - -
    -
    - - -
    -
    - - -
    -
    -
    - -
    - -
    -
    - - -
    -
    -
      -
    • - Provisional registration completion -
      • -
    • - We've sent a verification e-mail to the registered e-mail address. Verify your registration by confirming the message.
      • -
    • - If you do not receive the verification email, please check the following .
      • -
    -
      -
    • Or there was no error in - your input our e-mail address
      • -
    • Or e-mail from the " @ openrec.jp " domain has not been classified as junk folder
      • -
    • Or e-mail from the " @ openrec.jp " domain is not being blocked by domain rejection setting
    -
    - - -
    - -
    -
    - - -
    -
    -
      -
    • - Reregistration your password -
      • -
    • - Enter the email address you've registered.
      -We'll send you an email to reset the password.
      • -
    • - -
      • -
    • -
    • - -
      • -
    - -
    -
    - - -
    -
    -
    -
    - Reregistration your password
    -
    - We've sent you an email to reset the password.
    -
    - -
    - -
    -
    - - -
    -
    -
      -
    • - Reregistration your password -
      • -
    • - Highly secure password is a combination of alphanumeric characters and symbols .
      • -
    • - -
      • -
    • - -
      • -
    • -

      - Reregistration your password

      -
      • -
    - -
    -
    - - -
    -
    -
      -
    • - Report as a Inappropriate movie -
      • -
    • - Types of problem -
      • -
    • - -
      • -
    • - Details of problem -
      • -
    • - -
      • -
    • - We will check if reported contents is violating terms of service
      - To use this function immoderately is also violating terms of service.
      • -
    • -

      - Send

      -
      • -
    - -
    -
    - - -
    -
    -
      -
    • - Report as a spam -
      • -
    • - Details of problem -
      • -
    • - -
      • -
    • - Details of problem -
      • -
    • - -
      • -
    • - We will check if reported contents is violating terms of service
      - To use this function immoderately is also violating terms of service.
      • -
    • -

      - Send

      -
      • -
    - -
    -
    - - -
    -
    -
      -
    • - Report as a Inappropriate movie -
      • -
    • - Thank you for reporting
      - We will check if reported contents is violating terms of service
      -
      • -
    - -
    -
    - - -
    -
    -
      -
    • - Report as a spam -
      • -
    • - Thank you for reporting
      - We will check if reported contents is violating terms of service
      -
      • -
    - -
    -
    - -
    -
    -
      -
    • - For playing video -
      • -
    • - Playing video, it is required to install Flash Player. If you can't play video please download the latest version of Flash Player from here.
      • -
    - -
    -
    - - -
    -
    -
    -
    - Your request is pending now.
    Please wait for the result.
    -
    - Your request got denied.
    -
    - You can not apply with this account.
    -
    - -
    -
    - - - - -
    -
    -
    - - - -
    -
    -
      -
    • - To watch a previous live broadcast -
      • -
    • -
      You can get various privileges by upgrading to premium account
      -
      - Chat/comment stamp -
      -
      - Unlimited viewing of archived videos -
      -
      - Special Premium Member only broadcasts -
      -
      • -
    • - Register in only 3 steps!Easy ragestration.¥550/month
      • -
    • - Become premium member -
      • -
    - -
    -
    - - -
    -
    -
      -
    • - 超低遅延モードで視聴するためには、
      プレミアム会員への入会が必要です。       -
      • -
    • -
      You can get various privileges by upgrading to premium account
      -
      - Chat/comment stamp -
      -
      - Unlimited viewing of archived videos -
      -
      - Special Premium Member only broadcasts -
      -
      • -
    • - Register in only 3 steps!Easy ragestration.¥550/month
      • -
    • - Become premium member -
      • -
    - -
    -
    - - -
    -
    -
      -
    • - You can set 200 forbidden words
      if you upgrade to premium account       -
      • -
    • -
      You can get various privileges by upgrading to premium account
      -
      - Chat/comment stamp -
      -
      - Unlimited viewing of archived videos -
      -
      - Special Premium Member only broadcasts -
      -
      • -
    • - Register in only 3 steps!Easy ragestration.¥550/month
      • -
    • - Become premium member -
      • -
    - -
    -
    - - -
    -
    -
      -
    • - Premium user can pop out a chat. -
      • -
    • -
      You can get various privileges by upgrading to premium account
      -
      - Chat/comment stamp -
      -
      - Unlimited viewing of archived videos -
      -
      - Special Premium Member only broadcasts -
      -
      • -
    • - Register in only 3 steps!Easy ragestration.¥550/month
      • -
    • - Become premium member -
      • -
    - -
    -
    - -
    -
    -
    -
    - This video has been finished
    -
    - -
    - -
    -
    - - -
    -
    -
    -
    - Do you want to remove it from the blacklist?
    -
    - - -
    - -
    -
    - - -
    -
    -
    -
    - Are you sure you want to add to the black list?
    -
    - - -
    - -
    -
    - - -
    -
    -
    -
    - Do you want to repoart as a spam user?
    -
    - - -
    - -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    - -
    - -
    -
    -

    Videos

    -

    5

    -
    -
    -

    Views

    -

    732,990

    -
    -
    -

    Followers

    -

    64,978

    -
    -
    -

    Supporter

    -

    281

    -
    -
    - - -
    - -
    -
    - -
    -
    -
    -
    -
    - 手越ちゃんねる
    -
    -
    -
    -
    -
    -
    - -
    - -
    -
    -
    -

    Channels

     - -

    - Capture

    -     -

    Follow

     -

    Follower

     -

    Supporter

     -

    Your team

     -

    Playlist

     -
    -
    -
    - - - - - -
    -
    -
    -
    -
    -
    -

    -
    - -
    -
    -
    -
    - Are you sure to add to the team members
    -
    - - -
    - -
    -
    -
    -
    -
    -
    - Are you sure to remove from the team members
    -
    - - -
    - -
    -
    - -
    -
    -
    - - diff --git a/testfile/openrec_user_supporters.json b/testfile/openrec_user_supporters.json new file mode 100755 index 0000000..7bbb2a8 --- /dev/null +++ b/testfile/openrec_user_supporters.json @@ -0,0 +1,7 @@ +{ + "supporters": [ + "{\"status\":0,\"data\":{\"items\":[{\"user_id\":228479866,\"user_name\":\"Dayeee\",\"user_icon\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user\\/2284799\\/228479866.png?1590723285\",\"user_key\":\"shengyq1993\",\"user_bg\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"background_file\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"user_introduce\":\"\\u3074\\u3087\\u3093\\u3055\\u3093\\u306e\\u30d5\\u30a1\\u30f3\\u3067\\u3059\",\"user_type\":2,\"identify_id\":\"B_fON_LCTQ80x\",\"followed_flg\":0,\"movie_total_views\":null,\"onair_flg\":0,\"onair\":null,\"update_at\":null,\"total_yells\":10600,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/26\\/496dc6c54b4004437d44784f4c06921387d6e39c.gif\",\"supporter_rank\":1},{\"user_id\":180754796,\"user_name\":\"\\ud835\\udc24\\ud835\\udc1a\\ud835\\udc28\\ud835\\udc2b\\ud835\\udc22\\u2661\",\"user_icon\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user\\/1807548\\/180754796.png?1559139579\",\"user_key\":\"k11a18o\",\"user_bg\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user_background\\/1807548\\/180754796_cover.png?1584442115\",\"background_file\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user_background\\/1807548\\/180754796_cover.png?1584442115\",\"user_introduce\":\"\",\"user_type\":2,\"identify_id\":\"vmYmTocjdxMTK\",\"followed_flg\":0,\"movie_total_views\":null,\"onair_flg\":0,\"onair\":null,\"update_at\":null,\"total_yells\":7680,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/25\\/3496e3d762840443ccd55dd2e5809df0075f5173.gif\",\"supporter_rank\":2},{\"user_id\":84826105,\"user_name\":\"\\u3055\\u3044\\u3068\\u30fc\",\"user_icon\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user\\/848262\\/84826105.png?1513746953\",\"user_key\":\"saitoaaaaaaaa\",\"user_bg\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"background_file\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"user_introduce\":\"\",\"user_type\":2,\"identify_id\":\"XljOMaqesQ6ux\",\"followed_flg\":0,\"movie_total_views\":null,\"onair_flg\":0,\"onair\":null,\"update_at\":null,\"total_yells\":6800,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/25\\/3496e3d762840443ccd55dd2e5809df0075f5173.gif\",\"supporter_rank\":3},{\"user_id\":119385260,\"user_name\":\"\\u30a8\\u30c9\\u592a\",\"user_icon\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user\\/1193853\\/119385260.png?1593078041\",\"user_key\":\"yugata89\",\"user_bg\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user_background\\/1193853\\/119385260_cover.png?1535988951\",\"background_file\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user_background\\/1193853\\/119385260_cover.png?1535988951\",\"user_introduce\":\"\",\"user_type\":2,\"identify_id\":\"A5IawMLVQ8v2o\",\"followed_flg\":0,\"movie_total_views\":null,\"onair_flg\":0,\"onair\":null,\"update_at\":null,\"total_yells\":5840,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/25\\/3496e3d762840443ccd55dd2e5809df0075f5173.gif\",\"supporter_rank\":4},{\"user_id\":704857697,\"user_name\":\"H a n a\",\"user_icon\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user\\/7048577\\/704857697.png?1588750687\",\"user_key\":\"H_a_n_a_______\",\"user_bg\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"background_file\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"user_introduce\":\"\",\"user_type\":2,\"identify_id\":\"jBp4GE9X5BY\",\"followed_flg\":0,\"movie_total_views\":null,\"onair_flg\":0,\"onair\":null,\"update_at\":null,\"total_yells\":3000,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/24\\/a0dbf1fdc589aa12532afe1801d42d44f07bbb0c.gif\",\"supporter_rank\":5},{\"user_id\":194458029,\"user_name\":\"\\u3081\\u308b\",\"user_icon\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user\\/1944581\\/194458029.png?1564990460\",\"user_key\":\"pipi_oO\",\"user_bg\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"background_file\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"user_introduce\":\"\",\"user_type\":2,\"identify_id\":\"fUi4v1klHFnlg\",\"followed_flg\":0,\"movie_total_views\":null,\"onair_flg\":0,\"onair\":null,\"update_at\":null,\"total_yells\":2434,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/24\\/a0dbf1fdc589aa12532afe1801d42d44f07bbb0c.gif\",\"supporter_rank\":6},{\"user_id\":46792908,\"user_name\":\"\\u3074\\u3074\",\"user_icon\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user\\/467930\\/46792908.png?1512380327\",\"user_key\":\"youatto\",\"user_bg\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"background_file\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"user_introduce\":\"\",\"user_type\":2,\"identify_id\":\"NDg1hCUn8vzpP\",\"followed_flg\":0,\"movie_total_views\":null,\"onair_flg\":0,\"onair\":null,\"update_at\":null,\"total_yells\":2434,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/24\\/a0dbf1fdc589aa12532afe1801d42d44f07bbb0c.gif\",\"supporter_rank\":7},{\"user_id\":295982388,\"user_name\":\"\\u3053\\u3082\\u3082\\u3093\",\"user_icon\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user\\/2959824\\/295982388.png?1535135708\",\"user_key\":\"komomo0702\",\"user_bg\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user_background\\/2959824\\/295982388_cover.png?1555259860\",\"background_file\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user_background\\/2959824\\/295982388_cover.png?1555259860\",\"user_introduce\":\"\",\"user_type\":2,\"identify_id\":\"BaXhPdbhLdz_W\",\"followed_flg\":0,\"movie_total_views\":null,\"onair_flg\":0,\"onair\":null,\"update_at\":null,\"total_yells\":2400,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/24\\/a0dbf1fdc589aa12532afe1801d42d44f07bbb0c.gif\",\"supporter_rank\":8},{\"user_id\":4435320,\"user_name\":\"nono\",\"user_icon\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user\\/44354\\/4435320.png?1585839429\",\"user_key\":\"game19268033\",\"user_bg\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"background_file\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"user_introduce\":\"\",\"user_type\":2,\"identify_id\":\"ENgQDYpS1iyCR\",\"followed_flg\":0,\"movie_total_views\":null,\"onair_flg\":0,\"onair\":null,\"update_at\":null,\"total_yells\":2200,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/23\\/f77736048beb4c1b6fc7508c0c443ebafe44b851.gif\",\"supporter_rank\":9},{\"user_id\":721579501,\"user_name\":\"\\u3046\\u307f\",\"user_icon\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user\\/7215796\\/721579501.png?1593699619\",\"user_key\":\"yamamiu\",\"user_bg\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"background_file\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"user_introduce\":\"\",\"user_type\":2,\"identify_id\":\"egl2RPDrqbx\",\"followed_flg\":0,\"movie_total_views\":null,\"onair_flg\":0,\"onair\":null,\"update_at\":null,\"total_yells\":2200,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/23\\/f77736048beb4c1b6fc7508c0c443ebafe44b851.gif\",\"supporter_rank\":10},{\"user_id\":3930736,\"user_name\":\"\\u305f\\u3044\\u3061\\u3083\\u3093\\u306d\\u308b\",\"user_icon\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user\\/39308\\/3930736.png?1493878431\",\"user_key\":\"Yaritaiji\",\"user_bg\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user_background\\/39308\\/3930736_202005152205.png\",\"background_file\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user_background\\/39308\\/3930736_202005152205.png\",\"user_introduce\":\"\\u30b9\\u30d7\\u30e9\\u30c8\\u30a5\\u30fc\\u30f3\\u3068\\u304b\\u30b2\\u30fc\\u30e0\\u914d\\u4fe1\\u3057\\u307e\\u3059\\n\\nhttps:\\/\\/twitter.com\\/taijich0324\",\"user_type\":1,\"identify_id\":\"o3xBdEYzl37pC\",\"followed_flg\":0,\"movie_total_views\":\"83406087\",\"onair_flg\":0,\"onair\":null,\"update_at\":\"2020\\/7\\/13 21:37:29\",\"total_yells\":1600,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/23\\/f77736048beb4c1b6fc7508c0c443ebafe44b851.gif\",\"supporter_rank\":11},{\"user_id\":692929798,\"user_name\":\"aoi\",\"user_icon\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user\\/6929298\\/692929798.png?1592410193\",\"user_key\":\"aoi_1005_0723\",\"user_bg\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"background_file\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"user_introduce\":\"\",\"user_type\":2,\"identify_id\":\"VbMYjJv6vbv\",\"followed_flg\":0,\"movie_total_views\":null,\"onair_flg\":0,\"onair\":null,\"update_at\":null,\"total_yells\":1600,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/23\\/f77736048beb4c1b6fc7508c0c443ebafe44b851.gif\",\"supporter_rank\":12}]}}", + "{\"status\":0,\"data\":{\"items\":[{\"user_id\":498764657,\"user_name\":\"\\u3067\\u30fc\\u308b\",\"user_icon\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user\\/4987647\\/498764657.png?1588513790\",\"user_key\":\"chipdale33s\",\"user_bg\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"background_file\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"user_introduce\":\"\",\"user_type\":2,\"identify_id\":\"jAnaWaEJXOg\",\"followed_flg\":0,\"movie_total_views\":null,\"onair_flg\":0,\"onair\":null,\"update_at\":null,\"total_yells\":1600,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/23\\/f77736048beb4c1b6fc7508c0c443ebafe44b851.gif\",\"supporter_rank\":13},{\"user_id\":707167706,\"user_name\":\"\\u3072\\u308d\",\"user_icon\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user\\/7071678\\/707167706.png?1589063190\",\"user_key\":\"hiro_22\",\"user_bg\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"background_file\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"user_introduce\":\"\",\"user_type\":2,\"identify_id\":\"Ml3oVZy6rlD\",\"followed_flg\":0,\"movie_total_views\":null,\"onair_flg\":0,\"onair\":null,\"update_at\":null,\"total_yells\":1480,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/23\\/f77736048beb4c1b6fc7508c0c443ebafe44b851.gif\",\"supporter_rank\":14},{\"user_id\":244359955,\"user_name\":\"chiko\",\"user_icon\":null,\"user_key\":\"c_sugar5\",\"user_bg\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"background_file\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"user_introduce\":\"\",\"user_type\":2,\"identify_id\":\"9suM8ASMHRsHA\",\"followed_flg\":0,\"movie_total_views\":null,\"onair_flg\":0,\"onair\":null,\"update_at\":null,\"total_yells\":1200,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/23\\/f77736048beb4c1b6fc7508c0c443ebafe44b851.gif\",\"supporter_rank\":15},{\"user_id\":715018720,\"user_name\":\"\\u306a\\u306e\",\"user_icon\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user\\/7150188\\/715018720.png?1593430058\",\"user_key\":\"itigomusume\",\"user_bg\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"background_file\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"user_introduce\":\"\",\"user_type\":2,\"identify_id\":\"9X22kGnEAXx\",\"followed_flg\":0,\"movie_total_views\":null,\"onair_flg\":0,\"onair\":null,\"update_at\":null,\"total_yells\":1040,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/23\\/f77736048beb4c1b6fc7508c0c443ebafe44b851.gif\",\"supporter_rank\":16},{\"user_id\":700811078,\"user_name\":\"\\u307f\\u30fc\\u30fc\",\"user_icon\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user\\/7008111\\/700811078.png?1588158172\",\"user_key\":\"adgjmptw_258\",\"user_bg\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"background_file\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"user_introduce\":\"\",\"user_type\":2,\"identify_id\":\"ygG9Qn8dYbv\",\"followed_flg\":0,\"movie_total_views\":null,\"onair_flg\":0,\"onair\":null,\"update_at\":null,\"total_yells\":1040,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/23\\/f77736048beb4c1b6fc7508c0c443ebafe44b851.gif\",\"supporter_rank\":17},{\"user_id\":689988739,\"user_name\":\"\\u30de\\u30d5\\u30a3\\u30f3\",\"user_icon\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user\\/6899888\\/689988739.png?1594472502\",\"user_key\":\"wnwn_wdrdy\",\"user_bg\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"background_file\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"user_introduce\":\"\",\"user_type\":2,\"identify_id\":\"q19R7lwV9g8\",\"followed_flg\":0,\"movie_total_views\":null,\"onair_flg\":0,\"onair\":null,\"update_at\":null,\"total_yells\":1000,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/23\\/f77736048beb4c1b6fc7508c0c443ebafe44b851.gif\",\"supporter_rank\":18},{\"user_id\":694279147,\"user_name\":\"\\u3042\\u3044\\u3061\\u3083\\u3093\",\"user_icon\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user\\/6942792\\/694279147.png?1589038772\",\"user_key\":\"sasasama1112\",\"user_bg\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"background_file\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"user_introduce\":\"\\u9326\",\"user_type\":2,\"identify_id\":\"7oMWnqR9VJx\",\"followed_flg\":0,\"movie_total_views\":null,\"onair_flg\":0,\"onair\":null,\"update_at\":null,\"total_yells\":1000,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/23\\/f77736048beb4c1b6fc7508c0c443ebafe44b851.gif\",\"supporter_rank\":19},{\"user_id\":160426237,\"user_name\":\"\\u3086\\u304d\\u9178\",\"user_icon\":null,\"user_key\":\"yukisan62\",\"user_bg\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"background_file\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"user_introduce\":\"\",\"user_type\":2,\"identify_id\":\"sjobSgml8yFAU\",\"followed_flg\":0,\"movie_total_views\":null,\"onair_flg\":0,\"onair\":null,\"update_at\":null,\"total_yells\":800,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/23\\/f77736048beb4c1b6fc7508c0c443ebafe44b851.gif\",\"supporter_rank\":20},{\"user_id\":431849798,\"user_name\":\"\\u3048\\u307f\\u308a\\u3093\\u3068\\u3059\",\"user_icon\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user\\/4318498\\/431849798.png?1554369655\",\"user_key\":\"emirintoshhh\",\"user_bg\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"background_file\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"user_introduce\":\"\",\"user_type\":2,\"identify_id\":\"nOdm6o10DAQ\",\"followed_flg\":0,\"movie_total_views\":null,\"onair_flg\":0,\"onair\":null,\"update_at\":null,\"total_yells\":800,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/23\\/f77736048beb4c1b6fc7508c0c443ebafe44b851.gif\",\"supporter_rank\":21},{\"user_id\":745366562,\"user_name\":\"\\u30eb\\u30ab\",\"user_icon\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user\\/7453666\\/745366562.png?1593870385\",\"user_key\":\"1988rose\",\"user_bg\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"background_file\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"user_introduce\":null,\"user_type\":2,\"identify_id\":\"dgOWMJeGBg5\",\"followed_flg\":0,\"movie_total_views\":null,\"onair_flg\":0,\"onair\":null,\"update_at\":null,\"total_yells\":800,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/23\\/f77736048beb4c1b6fc7508c0c443ebafe44b851.gif\",\"supporter_rank\":22},{\"user_id\":195436432,\"user_name\":\"\\u306a\\u3063\\u3061\\u3083\\u3093\",\"user_icon\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user\\/1954365\\/195436432.png?1525131618\",\"user_key\":\"nacchaaaan\",\"user_bg\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"background_file\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"user_introduce\":\"\",\"user_type\":2,\"identify_id\":\"k0tYrSVxKKQpp\",\"followed_flg\":0,\"movie_total_views\":null,\"onair_flg\":0,\"onair\":null,\"update_at\":null,\"total_yells\":800,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/23\\/f77736048beb4c1b6fc7508c0c443ebafe44b851.gif\",\"supporter_rank\":23},{\"user_id\":154805243,\"user_name\":\"\\u3051\\u3044\",\"user_icon\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user\\/1548053\\/154805243.png?1550334800\",\"user_key\":\"keiko1018\",\"user_bg\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"background_file\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"user_introduce\":\"\",\"user_type\":2,\"identify_id\":\"voLRaZeS5o7we\",\"followed_flg\":0,\"movie_total_views\":null,\"onair_flg\":0,\"onair\":null,\"update_at\":null,\"total_yells\":800,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/23\\/f77736048beb4c1b6fc7508c0c443ebafe44b851.gif\",\"supporter_rank\":24}]}}", + "{\"status\":0,\"data\":{\"items\":[{\"user_id\":142527800,\"user_name\":\"\\u308b\\u3059\",\"user_icon\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user\\/1425278\\/142527800.png?1591854905\",\"user_key\":\"rusuninaritai\",\"user_bg\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user_background\\/1425278\\/142527800.png?1584450166\",\"background_file\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user_background\\/1425278\\/142527800.png?1584450166\",\"user_introduce\":\"[\\u5927\\u4f1a\\u5b9f\\u7e3e]\\n\\u7b2c13\\u56deglorycup \\u30d9\\u30b9\\u30c84\\n\\u7b2c2\\u56deNew!\\u30a8\\u30ea\\u30a2\\u676f \\u30d9\\u30b9\\u30c84\\n\\u7b2c8\\u56de\\u30a8\\u30ea\\u30a2\\u676f\\uff0b \\u30d9\\u30b9\\u30c84\\n\\u7b2c11\\u56deNew!\\u30a8\\u30ea\\u30a2\\u676f \\u30d9\\u30b9\\u30c84\\n\\u7b2c13\\u56deNew!\\u30a8\\u30ea\\u30a2\\u676f \\u30d9\\u30b9\\u30c83\\n\\u7b2c14\\u56deNew!\\u30a8\\u30ea\\u30a2\\u676f \\u6e96\\u512a\\u52dd\\n\\u7b2c16\\u56deNew!\\u30a8\\u30ea\\u30a2\\u676f \\u30d9\\u30b9\\u30c84\\n\\u7b2c18\\u56de\\u30a8\\u30ea\\u30a2\\u676f\\uff0b \\u30d9\\u30b9\\u30c83\\n\\u7b2c5\\u56de\\u30ca\\u30ef\\u30d0\\u30ea\\u676f \\u6e96\\u512a\\u52dd\\n\\u7b2c6\\u56de\\u30ca\\u30ef\\u30d0\\u30ea\\u676f \\u6e96\\u512a\\u52dd\\n\\u7b2c8\\u56de\\u30ca\\u30ef\\u30d0\\u30ea\\u676f \\u6e96\\u512a\\u52dd\\n\\u7b2c15\\u56de\\u7121\\u6575\\u7981\\u6b62\\u676f \\u512a\\u52dd\\n\\u7b2c1\\u56de\\u4e0a\\u4f4dtier\\u6b66\\u5668\\u7981\\u6b62\\u676f \\u512a\\u52dd\\n\\u30c0\\u30a4\\u30ca\\u30e2XP2974\",\"user_type\":1,\"identify_id\":\"Ntnpbs_bcU8l8\",\"followed_flg\":0,\"movie_total_views\":\"1685613\",\"onair_flg\":0,\"onair\":null,\"update_at\":\"2020\\/7\\/14 02:45:27\",\"total_yells\":640,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/22\\/2e6d5b4895ae00b656a700e2d25ef41e9e2600cd.gif\",\"supporter_rank\":25},{\"user_id\":536290679,\"user_name\":\"\\u3055\\u3093\\u3059\\u3051*\",\"user_icon\":null,\"user_key\":\"tktk-01234\",\"user_bg\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"background_file\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"user_introduce\":null,\"user_type\":2,\"identify_id\":\"98aLYLD27Ab\",\"followed_flg\":0,\"movie_total_views\":null,\"onair_flg\":0,\"onair\":null,\"update_at\":null,\"total_yells\":480,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/22\\/2e6d5b4895ae00b656a700e2d25ef41e9e2600cd.gif\",\"supporter_rank\":26},{\"user_id\":670887609,\"user_name\":\"\\u3061\\u3043\",\"user_icon\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user\\/6708877\\/670887609.png?1583749066\",\"user_key\":\"chii_1\",\"user_bg\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user_background\\/6708877\\/670887609_cover.png?1594455043\",\"background_file\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user_background\\/6708877\\/670887609_cover.png?1594455043\",\"user_introduce\":\"\\u914d\\u4fe1\\u3059\\u308b\\u304b\\u3057\\u306a\\u3044\\u304b\\u306f\\u74b0\\u5883\\u51fa\\u6765\\u3066\\u304b\\u3089\\u6c7a\\u3081\\u307e\\u3059\",\"user_type\":2,\"identify_id\":\"Ng4M4O6YNXq\",\"followed_flg\":0,\"movie_total_views\":null,\"onair_flg\":0,\"onair\":null,\"update_at\":null,\"total_yells\":480,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/22\\/2e6d5b4895ae00b656a700e2d25ef41e9e2600cd.gif\",\"supporter_rank\":27},{\"user_id\":3658258,\"user_name\":\"\\u3072\\u3044\\u3089\\u304e\",\"user_icon\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user\\/36583\\/3658258.png?1591548658\",\"user_key\":\"hiragispla\",\"user_bg\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user_background\\/36583\\/3658258_cover.png?1591547045\",\"background_file\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user_background\\/36583\\/3658258_cover.png?1591547045\",\"user_introduce\":\"Twitter\\u2192https:\\/\\/twitter.com\\/hiragi0223spla2\",\"user_type\":1,\"identify_id\":\"q_nlRPYAl4NOQ\",\"followed_flg\":0,\"movie_total_views\":\"997973\",\"onair_flg\":0,\"onair\":null,\"update_at\":\"2020\\/7\\/13 18:16:58\",\"total_yells\":400,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/22\\/2e6d5b4895ae00b656a700e2d25ef41e9e2600cd.gif\",\"supporter_rank\":28},{\"user_id\":113903074,\"user_name\":\"\\u9060\\u85e4\\u3061\\u304b\\u3057\",\"user_icon\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user\\/1139031\\/113903074.png?1593980363\",\"user_key\":\"yutorin_916\",\"user_bg\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user_background\\/1139031\\/113903074_cover.png?1590494029\",\"background_file\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user_background\\/1139031\\/113903074_cover.png?1590494029\",\"user_introduce\":\"Twitter @splar_ \\n\\u30a4\\u30f3\\u30b9\\u30bf aimyoncamper\\n\\u30e8\\u30ed\\u30b7\\u30af\\u30cd\",\"user_type\":1,\"identify_id\":\"h7yjbZhS4VIsb\",\"followed_flg\":0,\"movie_total_views\":\"244973\",\"onair_flg\":0,\"onair\":null,\"update_at\":\"2020\\/7\\/13 15:04:01\",\"total_yells\":400,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/22\\/2e6d5b4895ae00b656a700e2d25ef41e9e2600cd.gif\",\"supporter_rank\":29},{\"user_id\":641638275,\"user_name\":\"\\u3059\\u30fc\\u3061\\u3083\\u3093\",\"user_icon\":null,\"user_key\":\"suchan_\",\"user_bg\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"background_file\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"user_introduce\":\"\",\"user_type\":2,\"identify_id\":\"yOorwM5YrBE\",\"followed_flg\":0,\"movie_total_views\":null,\"onair_flg\":0,\"onair\":null,\"update_at\":null,\"total_yells\":400,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/22\\/2e6d5b4895ae00b656a700e2d25ef41e9e2600cd.gif\",\"supporter_rank\":30},{\"user_id\":110696888,\"user_name\":\"PANKO\",\"user_icon\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user\\/1106969\\/110696888.png?1587648021\",\"user_key\":\"sakupan06\",\"user_bg\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"background_file\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"user_introduce\":\"\",\"user_type\":2,\"identify_id\":\"hMYG9tGa0o1JG\",\"followed_flg\":0,\"movie_total_views\":null,\"onair_flg\":0,\"onair\":null,\"update_at\":null,\"total_yells\":400,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/22\\/2e6d5b4895ae00b656a700e2d25ef41e9e2600cd.gif\",\"supporter_rank\":31},{\"user_id\":676511312,\"user_name\":\"\\u307b \\u306a\",\"user_icon\":\"https:\\/\\/openrec-appdata.s3.amazonaws.com\\/user\\/6765114\\/676511312.png?1584621136\",\"user_key\":\"hona_126\",\"user_bg\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"background_file\":\"https:\\/\\/www.openrec.tv\\/viewapp\\/images\\/v8\\/img_back.png\",\"user_introduce\":\"\",\"user_type\":2,\"identify_id\":\"4Xr22JKr2Xp\",\"followed_flg\":0,\"movie_total_views\":null,\"onair_flg\":0,\"onair\":null,\"update_at\":null,\"total_yells\":400,\"yell_image_url\":\"https:\\/\\/dqd0jw5gvbchn.cloudfront.net\\/yell\\/22\\/2e6d5b4895ae00b656a700e2d25ef41e9e2600cd.gif\",\"supporter_rank\":32}]}}" + ] +} \ No newline at end of file From 75b1d9fb774ff782e6b567b1a14bbad9e066e03b Mon Sep 17 00:00:00 2001 From: eson Date: Wed, 15 Jul 2020 15:44:21 +0800 Subject: [PATCH 08/14] =?UTF-8?q?TODO:=20=E5=AE=8C=E5=96=84`=E4=B8=BB?= =?UTF-8?q?=E6=92=AD=E8=A1=A8`=E7=9A=84=E6=9B=B4=E6=96=B0=E6=96=B9?= =?UTF-8?q?=E5=BC=8F.=20=E5=AD=98=E5=9C=A8=E6=9B=B4=E6=96=B0=E6=9C=80?= =?UTF-8?q?=E6=96=B0=E7=8A=B6=E6=80=81=E6=97=B6,=20=E4=BC=9A=E8=A2=AB`?= =?UTF-8?q?=E6=94=B9=E7=89=88`=E7=B1=BB=E4=BC=BC=E6=83=85=E5=86=B5?= =?UTF-8?q?=E8=A6=86=E7=9B=96,=20=E4=BB=8E=E8=80=8C=E6=89=BE=E4=B8=8D?= =?UTF-8?q?=E5=88=B0=E5=8E=9F=E4=BF=A1=E6=81=AF=E6=95=B0=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- extractor/openrec/openrec_test.go | 219 +++++++++++--------- go.mod | 1 + sql/intimate_extractor.sql | 8 +- sql/intimate_source.sql | 2 +- store.go | 6 +- tasks/openrec/openrec_task2/task_openrec.go | 2 +- utils.go | 28 +++ 7 files changed, 163 insertions(+), 103 deletions(-) create mode 100644 utils.go diff --git a/extractor/openrec/openrec_test.go b/extractor/openrec/openrec_test.go index bb73e3c..cc2cf15 100644 --- a/extractor/openrec/openrec_test.go +++ b/extractor/openrec/openrec_test.go @@ -4,85 +4,85 @@ import ( "database/sql" "encoding/json" "intimate" + "io/ioutil" "log" "os" "regexp" "strconv" "strings" "testing" + "time" "github.com/474420502/hunter" "github.com/474420502/requests" + "github.com/lestrrat-go/libxml2" "github.com/tidwall/gjson" ) -func preNUm(data byte) int { - var mask byte = 0x80 - var num int = 0 - //8bit中首个0bit前有多少个1bits - for i := 0; i < 8; i++ { - if (data & mask) == mask { - num++ - mask = mask >> 1 - } else { - break - } - } - return num -} -func isUtf8(data []byte) bool { - i := 0 - for i < len(data) { - if (data[i] & 0x80) == 0x00 { - // 0XXX_XXXX - i++ - continue - } else if num := preNUm(data[i]); num > 2 { - // 110X_XXXX 10XX_XXXX - // 1110_XXXX 10XX_XXXX 10XX_XXXX - // 1111_0XXX 10XX_XXXX 10XX_XXXX 10XX_XXXX - // 1111_10XX 10XX_XXXX 10XX_XXXX 10XX_XXXX 10XX_XXXX - // 1111_110X 10XX_XXXX 10XX_XXXX 10XX_XXXX 10XX_XXXX 10XX_XXXX - // preNUm() 返回首个字节的8个bits中首个0bit前面1bit的个数，该数量也是该字符所使用的字节数 - i++ - for j := 0; j < num-1; j++ { - //判断后面的 num - 1 个字节是不是都是10开头 - if (data[i] & 0xc0) != 0x80 { - return false - } - i++ - } - } else { - //其他情况说明不是utf-8 - return false - } - } - return true +func TestCase1(t *testing.T) { + date := "2020-07-13T18:58:24+09:00" + + tm, err := time.Parse("2006-01-02T15:04:05Z07:00", date) + t.Error(err) + t.Error(time.Now()) + t.Error(tm.Local().UTC(), tm.Local()) + } -func isGBK(data []byte) bool { - length := len(data) - var i int = 0 - for i < length { - if data[i] <= 0x7f { - //编码0~127,只有一个字节的编码，兼容ASCII码 - i++ - continue - } else { - //大于127的使用双字节编码，落在gbk编码范围内的字符 - if data[i] >= 0x81 && - data[i] <= 0xfe && - data[i+1] >= 0x40 && - data[i+1] <= 0xfe && - data[i+1] != 0xf7 { - i += 2 - continue - } else { - return false - } - } +func TestCase2(t *testing.T) { + duration1 := "0:00:00" + duration2 := "4:56:04" + tm2, err := time.Parse("15:04:05", duration2) + tm1, err := time.Parse("15:04:05", duration1) + + tm2.Sub(tm1) + + t.Error(err) + t.Error(tm2.Sub(tm1)) + +} + +func TestCase(t *testing.T) { + f, _ := os.Open("./test.html") + data, _ := ioutil.ReadAll(f) + + doc, err := libxml2.ParseHTML(data) + if err != nil { + panic(err) } - return true + // doc.CreateElement("meta") + // "" + + xresult, err := doc.Find("/html/head") + ele, err := doc.CreateElement(`META`) + + if err != nil { + panic(err) + } + ele.SetAttribute("charset", "utf-8") + + if err != nil { + panic(err) + } + + iter := xresult.NodeIter() + if iter.Next() { + n := iter.Node() + + err = n.AddChild(ele) + // childs, err := n.ChildNodes() + if err != nil { + t.Error(err) + } + t.Error(n) + } + + xr, err := doc.Find("//h1[ contains(@class, 'MovieTitle__Title')]") + if err != nil { + panic(nil) + } + + t.Error(xr) } func TestExtractor(t *testing.T) { @@ -96,6 +96,15 @@ func TestExtractor(t *testing.T) { collect := intimate.NewExtractorStore() store := intimate.NewSourceStore("source_openrec") source, err := store.Pop(string(intimate.TTOpenrecRanking), 100) + + anchorId := source.GetSource().String + + ai := &intimate.AnchorInfo{} + ai.SetAnchorId(anchorId) + ai.SetPlatform(string(intimate.Popenrec)) + + collect.InsertAnchorInfo(ai) + // if source != nil { // defer store.Restore(source) // } @@ -110,7 +119,7 @@ func TestExtractor(t *testing.T) { user := m["user"] - ai := &intimate.CollectLog{} + clog := &intimate.CollectLog{} extractor := hunter.NewExtractor([]byte(user.Str)) xp, err := extractor.XPathResult("//p[@class='c-global__user__count__row__right js-userCountFollowers']/text()") if err != nil { @@ -121,8 +130,8 @@ func TestExtractor(t *testing.T) { } followers := strings.ReplaceAll(xp.String(), ",", "") - followersInt, err := strconv.ParseInt(followers, 10, 64) + if err != nil { t.Error(err) } @@ -134,6 +143,7 @@ func TestExtractor(t *testing.T) { } else { t.Error(err) } + t.Error(source.GetSource()) t.Error(anchorName) // c-contents @@ -149,8 +159,8 @@ func TestExtractor(t *testing.T) { t.Error(err) } - ai.SetViews(sql.NullInt64{Int64: int64(viewsint), Valid: true}) - ai.SetIsShowing(1) + clog.SetViews(sql.NullInt64{Int64: int64(viewsint), Valid: true}) + clog.SetIsShowing(1) } var givers []interface{} @@ -167,50 +177,69 @@ func TestExtractor(t *testing.T) { giversbytes, err := json.Marshal(givers) if err != nil { t.Error(err) - ai.SetErrorMsg(sql.NullString{String: err.Error(), Valid: true}) + clog.SetErrorMsg(sql.NullString{String: err.Error(), Valid: true}) } else { - ai.SetGiver(giversbytes) + clog.SetGiver(giversbytes) } // MovieToolbar__Views-g5e6ic-13 iDRGyA livejson := m["user_live"] - f, err := os.OpenFile("./test.html", os.O_CREATE|os.O_TRUNC|os.O_RDWR, os.ModePerm) - if err != nil { - panic(err) - } - f.WriteString(livejson.String()) - t.Error(livejson) - extractor = hunter.NewExtractor([]byte(livejson.Str)) - xr, err := extractor.XPathResult("//h1[ contains(@class, 'MovieTitle__Title')]") + // f, err := os.OpenFile("./test.html", os.O_CREATE|os.O_TRUNC|os.O_RDWR, os.ModePerm) + // if err != nil { + // panic(err) + // } + // f.WriteString(livejson.String()) - if err != nil { - t.Error(err) - } - t.Error(xr) - iter := xr.NodeIter() - if iter.Next() { - t.Error(iter.Node().TextContent()) - ai.SetShowTitle(sql.NullString{String: iter.Node().TextContent(), Valid: true}) + extractor = hunter.NewExtractor([]byte(livejson.Str)) + // xr, err := extractor.XPathResult("//h1[ contains(@class, 'MovieTitle__Title')]") + // if err != nil { + // t.Error(err) + // } + + mathes := regexp.MustCompile("MovieTitle__Title.*>(.+)").FindStringSubmatch(livejson.Str) + if len(mathes) == 2 { + + clog.SetShowTitle(sql.NullString{String: mathes[1], Valid: true}) content, err := extractor.XPathResult("//meta[@itemprop='uploadDate']/@content") if err != nil { t.Error(err) } - if content.NodeIter().Next() { - t.Error(content.String()) + iter := content.NodeIter() + if iter.Next() { + tm, err := time.ParseInLocation("2006-01-02T15:04:05Z07:00", iter.Node().NodeValue(), time.Local) + if err != nil { + t.Error(err) + } + clog.SetShowStartTime(sql.NullTime{Time: tm.Local(), Valid: true}) + + duration, err := extractor.XPathResult("//meta[@itemprop='duration']/@content") + if err != nil { + t.Error(err) + } + + diter := duration.NodeIter() + if diter.Next() { + + dt, err := intimate.ParseDuration(diter.Node().NodeValue()) + if err != nil { + log.Println(err) + } + endtm := tm.Add(dt) + clog.SetShowEndTime(sql.NullTime{Time: endtm.Local(), Valid: true}) + } } } - t.Error(xr.String(), xr.NodeIter().Next(), xr.String()) - ai.SetGratuity(sql.NullInt64{Int64: gratuity, Valid: true}) - ai.SetPlatform(string(intimate.Popenrec)) - ai.SetFollowers(sql.NullInt64{Int64: int64(followersInt), Valid: true}) - ai.SetAnchorId(source.GetSource().String) - ai.SetUpdateTime(source.GetUpdateTime()) + clog.SetGratuity(sql.NullInt64{Int64: gratuity, Valid: true}) + clog.SetPlatform(string(intimate.Popenrec)) + clog.SetFollowers(sql.NullInt64{Int64: int64(followersInt), Valid: true}) + clog.SetAnchorId(anchorId) + clog.SetUpdateTime(source.GetUpdateTime()) - collect.InsertCollectLog(ai) + collect.InsertCollectLog(clog) } else { t.Error("data is not json:\n", string(sdata)) diff --git a/go.mod b/go.mod index 6575876..690761f 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/474420502/hunter v0.3.0 github.com/474420502/requests v1.6.0 github.com/go-sql-driver/mysql v1.5.0 + github.com/lestrrat-go/libxml2 v0.0.0-20200215080510-6483566f52cb github.com/tidwall/gjson v1.6.0 github.com/tidwall/pretty v1.0.1 // indirect golang.org/x/net v0.0.0-20200707034311-ab3426394381 // indirect diff --git a/sql/intimate_extractor.sql b/sql/intimate_extractor.sql index cb1d1e5..67a2d03 100644 --- a/sql/intimate_extractor.sql +++ b/sql/intimate_extractor.sql @@ -10,7 +10,7 @@ CREATE TABLE IF NOT EXISTS `anchor_info` ( `channel` varchar(128) DEFAULT NULL, `show_type` varchar(255) DEFAULT NULL, `ext` json DEFAULT NULL, - `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, PRIMARY KEY (`uid`), UNIQUE KEY `platform_anchor_id_idx` (`platform`, `anchor_id`), KEY `platform_idx` (`platform`), @@ -35,9 +35,9 @@ CREATE TABLE IF NOT EXISTS `collect_log` ( `gratuity` bigint(11) DEFAULT NULL, `show_title` text DEFAULT NULL, - `show_start_time` timestamp NULL DEFAULT NULL, - `show_end_time` timestamp NULL DEFAULT NULL, - `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `show_start_time` timestamp NULL DEFAULT NULL, + `show_end_time` timestamp NULL DEFAULT NULL, + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, `ext` json DEFAULT NULL, `error_msg` text DEFAULT NULL, diff --git a/sql/intimate_source.sql b/sql/intimate_source.sql index fac4d51..8c489a3 100644 --- a/sql/intimate_source.sql +++ b/sql/intimate_source.sql @@ -8,7 +8,7 @@ CREATE TABLE IF NOT EXISTS `source_openrec` ( `source` longtext DEFAULT NULL, `ext` json DEFAULT NULL, `pass_gob` blob DEFAULT NULL, - `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, `operator` int DEFAULT 0, `error_msg` text DEFAULT NULL, PRIMARY KEY(`uid`), diff --git a/store.go b/store.go index 4810691..e99ecc9 100644 --- a/store.go +++ b/store.go @@ -181,9 +181,10 @@ func NewExtractorStore() *ExtractorStore { */ // InsertAnchorInfo AnchorInfo表, 插入数据 -func (store *ExtractorStore) InsertAnchorInfo(isource IGetAnchorInfo) { +func (store *ExtractorStore) InsertAnchorInfo(isource IGetAnchorInfo) error { _, err := store.db.Exec("insert into "+AnchorTable+"(platform, anchor_id, anchor_name, live_url, channel, show_type, ext) values(?,?,?,?,?,?,?) ON DUPLICATE KEY UPDATE", isource.GetPlatform(), isource.GetAnchorId(), isource.GetAnchorName(), isource.GetLiveUrl(), isource.GetChannel(), isource.GetShowType(), isource.GetExt()) store.errorAlarm(err) + return err } /* @@ -209,9 +210,10 @@ func (store *ExtractorStore) InsertAnchorInfo(isource IGetAnchorInfo) { */ // InsertCollectLog CollectLog表插入数据 -func (store *ExtractorStore) InsertCollectLog(isource IGetCollectLog) { +func (store *ExtractorStore) InsertCollectLog(isource IGetCollectLog) error { _, err := store.db.Exec("insert into "+CollectLogTable+"(uid, platform, anchor_id, is_showing, is_error, followers, views, giver, gratuity, show_title, show_start_time, show_end_time, update_time, ext, error_msg) values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", isource.GetUid(), isource.GetPlatform(), isource.GetAnchorId(), isource.GetIsShowing(), isource.GetIsError(), isource.GetFollowers(), isource.GetViews(), isource.GetGiver(), isource.GetGratuity(), isource.GetShowTitle(), isource.GetShowStartTime(), isource.GetShowEndTime(), isource.GetUpdateTime(), isource.GetExt(), isource.GetErrorMsg(), ) store.errorAlarm(err) + return err } diff --git a/tasks/openrec/openrec_task2/task_openrec.go b/tasks/openrec/openrec_task2/task_openrec.go index 6610541..286914c 100644 --- a/tasks/openrec/openrec_task2/task_openrec.go +++ b/tasks/openrec/openrec_task2/task_openrec.go @@ -35,7 +35,7 @@ func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) { source, err := store.Pop(string(intimate.TTOpenrecUser)) - if source == nil && err != nil { + if source == nil || err != nil { log.Println(err) time.Sleep(time.Second * 2) continue diff --git a/utils.go b/utils.go new file mode 100644 index 0000000..f922bf5 --- /dev/null +++ b/utils.go @@ -0,0 +1,28 @@ +package intimate + +import ( + "log" + "time" +) + +var zeroTime time.Time + +func init() { + + tm, err := time.Parse("15:04:05", "0:00:00") + if err != nil { + log.Println(err) + } + zeroTime = tm + +} + +// ParseDuration time to duration eg: 1:40:00 -> time.Duration +func ParseDuration(dt string) (time.Duration, error) { + tdt, err := time.Parse("15:04:05", dt) + if err != nil { + + return time.Duration(0), err + } + return tdt.Sub(zeroTime), nil +} From b63e180499b87cffb42b0453de8da6f1e60fabb1 Mon Sep 17 00:00:00 2001 From: eson Date: Wed, 15 Jul 2020 18:22:40 +0800 Subject: [PATCH 09/14] =?UTF-8?q?=E6=B7=BB=E5=8A=A0tags,=20=E4=BF=AE?= =?UTF-8?q?=E6=94=B9=20=E5=AD=98=E5=82=A8=E7=BB=93=E6=9E=84=20=E6=9B=B4?= =?UTF-8?q?=E5=8A=A0=E6=8E=A5=E8=BF=91=E5=88=B6=E7=9B=B4=E6=92=AD=E7=9A=84?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E5=AD=98=E5=82=A8=E6=9E=B6=E6=9E=84.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- extractor/openrec/openrec_test.go | 21 ++++++++------ sql/intimate_extractor.sql | 2 +- store.go | 46 ++++++++++++++++++++++++++++--- 3 files changed, 56 insertions(+), 13 deletions(-) diff --git a/extractor/openrec/openrec_test.go b/extractor/openrec/openrec_test.go index cc2cf15..74eb55a 100644 --- a/extractor/openrec/openrec_test.go +++ b/extractor/openrec/openrec_test.go @@ -103,14 +103,6 @@ func TestExtractor(t *testing.T) { ai.SetAnchorId(anchorId) ai.SetPlatform(string(intimate.Popenrec)) - collect.InsertAnchorInfo(ai) - - // if source != nil { - // defer store.Restore(source) - // } - if err != nil { - t.Error(err) - } sdata := source.GetExt().([]byte) if gjson.ValidBytes(sdata) { @@ -143,9 +135,12 @@ func TestExtractor(t *testing.T) { } else { t.Error(err) } + t.Error(source.GetSource()) t.Error(anchorName) + ai.SetAnchorName(anchorName) + // c-contents xp, err = extractor.XPathResult("//ul[@class='c-contents']//p[@class='c-thumbnailVideo__footer__liveCount']/text()") if err != nil { @@ -233,6 +228,16 @@ func TestExtractor(t *testing.T) { } } + LiveUrl := "https://www.openrec.tv/live/" + anchorId + ai.SetLiveUrl(sql.NullString{String: LiveUrl, Valid: true}) + + Uid, err := collect.InsertAnchorInfo(ai) + if err != nil { + t.Error(err) + return + } + + clog.SetUid(Uid) clog.SetGratuity(sql.NullInt64{Int64: gratuity, Valid: true}) clog.SetPlatform(string(intimate.Popenrec)) clog.SetFollowers(sql.NullInt64{Int64: int64(followersInt), Valid: true}) diff --git a/sql/intimate_extractor.sql b/sql/intimate_extractor.sql index 67a2d03..bac815a 100644 --- a/sql/intimate_extractor.sql +++ b/sql/intimate_extractor.sql @@ -8,7 +8,7 @@ CREATE TABLE IF NOT EXISTS `anchor_info` ( `anchor_name` varchar(255) NOT NULL, `live_url` text, `channel` varchar(128) DEFAULT NULL, - `show_type` varchar(255) DEFAULT NULL, + `tags` json DEFAULT NULL, `ext` json DEFAULT NULL, `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, PRIMARY KEY (`uid`), diff --git a/store.go b/store.go index e99ecc9..904f4a4 100644 --- a/store.go +++ b/store.go @@ -181,12 +181,50 @@ func NewExtractorStore() *ExtractorStore { */ // InsertAnchorInfo AnchorInfo表, 插入数据 -func (store *ExtractorStore) InsertAnchorInfo(isource IGetAnchorInfo) error { - _, err := store.db.Exec("insert into "+AnchorTable+"(platform, anchor_id, anchor_name, live_url, channel, show_type, ext) values(?,?,?,?,?,?,?) ON DUPLICATE KEY UPDATE", isource.GetPlatform(), isource.GetAnchorId(), isource.GetAnchorName(), isource.GetLiveUrl(), isource.GetChannel(), isource.GetShowType(), isource.GetExt()) - store.errorAlarm(err) - return err +func (store *ExtractorStore) InsertAnchorInfo(isource IGetAnchorInfo) (Uid int64, err error) { + // select uid from table where platform = ? and anchor_id = ? + selectSQL := "select uid from " + AnchorTable + " where platform = ? and anchor_id = ?" + tx, err := store.db.Begin() + if err != nil { + log.Println(err) + return 0, err + } + + row := tx.QueryRow(selectSQL+` limit 1 for update`, isource.GetPlatform(), isource.GetAnchorId()) + + if row != nil { + var uid int64 + row.Scan(&uid) + return uid, nil + } + + result, err := tx.Exec("insert into "+AnchorTable+"(platform, anchor_id, anchor_name, live_url, channel, show_type, ext) values(?,?,?,?,?,?,?);", isource.GetPlatform(), isource.GetAnchorId(), isource.GetAnchorName(), isource.GetLiveUrl(), isource.GetChannel(), isource.GetShowType(), isource.GetExt()) + log.Println(result.LastInsertId()) + + if err != nil { + log.Println(err) + return 0, nil + } + + err = tx.Commit() + if err != nil { + log.Println(err) + err = tx.Rollback() + if err != nil { + log.Println(err) + } + } + + return result.LastInsertId() } +// InsertAnchorInfo AnchorInfo表, 插入数据 +// func (store *ExtractorStore) InsertAnchorInfo(isource IGetAnchorInfo) error { +// _, err := store.db.Exec("insert into "+AnchorTable+"(platform, anchor_id, anchor_name, live_url, channel, show_type, ext) values(?,?,?,?,?,?,?)", isource.GetPlatform(), isource.GetAnchorId(), isource.GetAnchorName(), isource.GetLiveUrl(), isource.GetChannel(), isource.GetShowType(), isource.GetExt()) +// store.errorAlarm(err) +// return err +// } + /* `uid` bigint, `platform` varchar(255) NOT NULL, From 13ae890171911f6c6c22e6edb7014a4e4da2af54 Mon Sep 17 00:00:00 2001 From: eson Date: Wed, 15 Jul 2020 19:23:45 +0800 Subject: [PATCH 10/14] =?UTF-8?q?TODO:=20=E6=AD=A3=E5=88=99Regexp=20?= =?UTF-8?q?=E8=8E=B7=E5=8F=96Tags=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- extractor/openrec/.gitignore | 2 + extractor/openrec/openrec_test.go | 17 ++++--- extractor_field.go | 75 ++++++++++++++++++------------- sql/intimate_extractor.sql | 2 +- store.go | 6 +-- 5 files changed, 62 insertions(+), 40 deletions(-) create mode 100644 extractor/openrec/.gitignore diff --git a/extractor/openrec/.gitignore b/extractor/openrec/.gitignore new file mode 100644 index 0000000..589ccd4 --- /dev/null +++ b/extractor/openrec/.gitignore @@ -0,0 +1,2 @@ +*.html +screenlog.* \ No newline at end of file diff --git a/extractor/openrec/openrec_test.go b/extractor/openrec/openrec_test.go index 74eb55a..33cb210 100644 --- a/extractor/openrec/openrec_test.go +++ b/extractor/openrec/openrec_test.go @@ -96,6 +96,10 @@ func TestExtractor(t *testing.T) { collect := intimate.NewExtractorStore() store := intimate.NewSourceStore("source_openrec") source, err := store.Pop(string(intimate.TTOpenrecRanking), 100) + if err != nil { + log.Println(err) + return + } anchorId := source.GetSource().String @@ -180,11 +184,11 @@ func TestExtractor(t *testing.T) { // MovieToolbar__Views-g5e6ic-13 iDRGyA livejson := m["user_live"] - // f, err := os.OpenFile("./test.html", os.O_CREATE|os.O_TRUNC|os.O_RDWR, os.ModePerm) - // if err != nil { - // panic(err) - // } - // f.WriteString(livejson.String()) + f, err := os.OpenFile("./test.html", os.O_CREATE|os.O_TRUNC|os.O_RDWR, os.ModePerm) + if err != nil { + panic(err) + } + f.WriteString(livejson.String()) extractor = hunter.NewExtractor([]byte(livejson.Str)) // xr, err := extractor.XPathResult("//h1[ contains(@class, 'MovieTitle__Title')]") @@ -228,6 +232,9 @@ func TestExtractor(t *testing.T) { } } + matheslist := regexp.MustCompile(`TagButton__Button.+>([^<]+)<`).FindAllStringSubmatch(livejson.Str, 0) + t.Error(matheslist) + LiveUrl := "https://www.openrec.tv/live/" + anchorId ai.SetLiveUrl(sql.NullString{String: LiveUrl, Valid: true}) diff --git a/extractor_field.go b/extractor_field.go index ad5127e..9721e1f 100644 --- a/extractor_field.go +++ b/extractor_field.go @@ -6,27 +6,27 @@ import ( ) type ISetAnchorInfo interface { - SetUid(int64) // - SetPlatform(string) // - SetAnchorId(string) // - SetAnchorName(string) // - SetLiveUrl(sql.NullString) // - SetChannel(sql.NullString) // - SetShowType(sql.NullString) // - SetExt(interface{}) // - SetUpdateTime(time.Time) // + SetUid(int64) // + SetPlatform(string) // + SetAnchorId(string) // + SetAnchorName(string) // + SetLiveUrl(sql.NullString) // + SetChannel(sql.NullString) // + SetTags(interface{}) // + SetExt(interface{}) // + SetUpdateTime(time.Time) // } type IGetAnchorInfo interface { - GetUid() int64 // - GetPlatform() string // - GetAnchorId() string // - GetAnchorName() string // - GetLiveUrl() sql.NullString // - GetChannel() sql.NullString // - GetShowType() sql.NullString // - GetExt() interface{} // - GetUpdateTime() time.Time // + GetUid() int64 // + GetPlatform() string // + GetAnchorId() string // + GetAnchorName() string // + GetLiveUrl() sql.NullString // + GetChannel() sql.NullString // + GetTags() interface{} + GetExt() interface{} // + GetUpdateTime() time.Time // } /* @@ -58,9 +58,19 @@ type AnchorInfo struct { AnchorName string // LiveUrl sql.NullString // Channel sql.NullString // - ShowType sql.NullString // - Ext interface{} // - UpdateTime time.Time // + Tags interface{} + Ext interface{} // + UpdateTime time.Time // +} + +// GetTags Get return Tags interface{} +func (ai *AnchorInfo) GetTags() interface{} { + return ai.Tags +} + +// SetTags Set Tags interface{} +func (ai *AnchorInfo) SetTags(Tags interface{}) { + ai.Tags = Tags } // GetUpdateTime Get return UpdateTime time.Time @@ -83,16 +93,6 @@ func (ai *AnchorInfo) SetExt(Ext interface{}) { ai.Ext = Ext } -// GetShowType Get return ShowType sql.NullString -func (ai *AnchorInfo) GetShowType() sql.NullString { - return ai.ShowType -} - -// SetShowType Set ShowType sql.NullString -func (ai *AnchorInfo) SetShowType(ShowType sql.NullString) { - ai.ShowType = ShowType -} - // GetChannel Get return Channel sql.NullString func (ai *AnchorInfo) GetChannel() sql.NullString { return ai.Channel @@ -167,6 +167,7 @@ type IGetCollectLog interface { GetShowStartTime() sql.NullTime // GetShowEndTime() sql.NullTime // GetUpdateTime() sql.NullTime // + GetTags() interface{} // GetExt() interface{} // GetErrorMsg() sql.NullString // } @@ -185,6 +186,7 @@ type ISetCollectLog interface { SetShowStartTime(sql.NullTime) // SetShowEndTime(sql.NullTime) // SetUpdateTime(sql.NullTime) // + SetTags(interface{}) // SetExt(interface{}) // SetErrorMsg(sql.NullString) // } @@ -237,10 +239,21 @@ type CollectLog struct { ShowStartTime sql.NullTime // ShowEndTime sql.NullTime // UpdateTime sql.NullTime // + Tags interface{} Ext interface{} // ErrorMsg sql.NullString // } +// GetTags Get return Tags interface{} +func (cl *CollectLog) GetTags() interface{} { + return cl.Tags +} + +// SetTags Set Tags interface{} +func (cl *CollectLog) SetTags(Tags interface{}) { + cl.Tags = Tags +} + // GetErrorMsg Get return Error sql.NullString func (cl *CollectLog) GetErrorMsg() sql.NullString { return cl.ErrorMsg diff --git a/sql/intimate_extractor.sql b/sql/intimate_extractor.sql index bac815a..6402f45 100644 --- a/sql/intimate_extractor.sql +++ b/sql/intimate_extractor.sql @@ -17,7 +17,6 @@ CREATE TABLE IF NOT EXISTS `anchor_info` ( KEY `anchor_id_idx` (`anchor_id`), KEY `anchor_name_idx` (`anchor_name`), KEY `channel_idx` (`channel`), - KEY `show_type_idx` (`show_type`), KEY `update_time_idx` (`update_time`) ); @@ -38,6 +37,7 @@ CREATE TABLE IF NOT EXISTS `collect_log` ( `show_start_time` timestamp NULL DEFAULT NULL, `show_end_time` timestamp NULL DEFAULT NULL, `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `tags` json DEFAULT NULL, `ext` json DEFAULT NULL, `error_msg` text DEFAULT NULL, diff --git a/store.go b/store.go index 904f4a4..48b30c0 100644 --- a/store.go +++ b/store.go @@ -198,7 +198,7 @@ func (store *ExtractorStore) InsertAnchorInfo(isource IGetAnchorInfo) (Uid int64 return uid, nil } - result, err := tx.Exec("insert into "+AnchorTable+"(platform, anchor_id, anchor_name, live_url, channel, show_type, ext) values(?,?,?,?,?,?,?);", isource.GetPlatform(), isource.GetAnchorId(), isource.GetAnchorName(), isource.GetLiveUrl(), isource.GetChannel(), isource.GetShowType(), isource.GetExt()) + result, err := tx.Exec("insert into "+AnchorTable+"(platform, anchor_id, anchor_name, live_url, channel, tags, ext) values(?,?,?,?,?,?,?);", isource.GetPlatform(), isource.GetAnchorId(), isource.GetAnchorName(), isource.GetLiveUrl(), isource.GetChannel(), isource.GetTags(), isource.GetExt()) log.Println(result.LastInsertId()) if err != nil { @@ -249,8 +249,8 @@ func (store *ExtractorStore) InsertAnchorInfo(isource IGetAnchorInfo) (Uid int64 // InsertCollectLog CollectLog表插入数据 func (store *ExtractorStore) InsertCollectLog(isource IGetCollectLog) error { - _, err := store.db.Exec("insert into "+CollectLogTable+"(uid, platform, anchor_id, is_showing, is_error, followers, views, giver, gratuity, show_title, show_start_time, show_end_time, update_time, ext, error_msg) values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", - isource.GetUid(), isource.GetPlatform(), isource.GetAnchorId(), isource.GetIsShowing(), isource.GetIsError(), isource.GetFollowers(), isource.GetViews(), isource.GetGiver(), isource.GetGratuity(), isource.GetShowTitle(), isource.GetShowStartTime(), isource.GetShowEndTime(), isource.GetUpdateTime(), isource.GetExt(), isource.GetErrorMsg(), + _, err := store.db.Exec("insert into "+CollectLogTable+"(uid, platform, anchor_id, is_showing, is_error, followers, views, giver, gratuity, show_title, show_start_time, show_end_time, update_time, tags, ext, error_msg) values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", + isource.GetUid(), isource.GetPlatform(), isource.GetAnchorId(), isource.GetIsShowing(), isource.GetIsError(), isource.GetFollowers(), isource.GetViews(), isource.GetGiver(), isource.GetGratuity(), isource.GetShowTitle(), isource.GetShowStartTime(), isource.GetShowEndTime(), isource.GetUpdateTime(), isource.GetTags(), isource.GetExt(), isource.GetErrorMsg(), ) store.errorAlarm(err) return err From 51fe6f6039fe0ac4a5f0c02b6736de146aaa7aa8 Mon Sep 17 00:00:00 2001 From: eson Date: Thu, 16 Jul 2020 11:02:30 +0800 Subject: [PATCH 11/14] =?UTF-8?q?finish:=20=E8=A7=A3=E6=9E=90=E9=80=BB?= =?UTF-8?q?=E8=BE=91,=20=E5=85=A5=E5=BA=93=E6=AD=A3=E7=A1=AE.=20TODO:=20?= =?UTF-8?q?=E6=95=B4=E7=90=86=E4=BB=A3=E7=A0=81,=20=E8=AE=A9=E5=85=A5?= =?UTF-8?q?=E5=BA=93=E6=8F=90=E5=8F=96=E6=95=B0=E6=8D=AE=E6=88=90=E4=B8=BA?= =?UTF-8?q?=20=E5=9F=BA=E7=A1=80=E5=BA=93.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- extractor/openrec/openrec_test.go | 32 +++++++++++++++++++++++--- extractor_field.go | 31 +++++++++++++------------- store.go | 37 +++++++++++-------------------- 3 files changed, 57 insertions(+), 43 deletions(-) diff --git a/extractor/openrec/openrec_test.go b/extractor/openrec/openrec_test.go index 33cb210..3c7eb9f 100644 --- a/extractor/openrec/openrec_test.go +++ b/extractor/openrec/openrec_test.go @@ -19,6 +19,20 @@ import ( "github.com/tidwall/gjson" ) +func TestCase0(t *testing.T) { + f, err := os.Open("./test.html") + if err != nil { + panic(err) + } + data, err := ioutil.ReadAll(f) + if err != nil { + panic(err) + } + + matheslist := regexp.MustCompile(`TagButton__Button[^>]+>(.{1,100})(.+)").FindStringSubmatch(livejson.Str) + mathes := regexp.MustCompile("MovieTitle__Title[^>]+>(.{1,50})").FindStringSubmatch(livejson.Str) if len(mathes) == 2 { clog.SetShowTitle(sql.NullString{String: mathes[1], Valid: true}) @@ -232,8 +246,19 @@ func TestExtractor(t *testing.T) { } } - matheslist := regexp.MustCompile(`TagButton__Button.+>([^<]+)<`).FindAllStringSubmatch(livejson.Str, 0) - t.Error(matheslist) + var tags []string + matheslist := regexp.MustCompile(`TagButton__Button[^>]+>(.{1,100}) Date: Thu, 16 Jul 2020 15:25:55 +0800 Subject: [PATCH 12/14] =?UTF-8?q?=E5=AE=8C=E6=88=90=E8=A7=A3=E6=9E=90?= =?UTF-8?q?=E6=A8=A1=E5=9D=97=E7=A4=BA=E4=BE=8B.=20=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E5=BA=93=E5=85=A5=E5=BA=93=E6=B5=8B=E8=AF=95.=20TODO:=20?= =?UTF-8?q?=E8=B0=83=E6=95=B4=E7=A8=8B=E5=BA=8F=E5=90=AF=E5=8A=A8=E5=81=9C?= =?UTF-8?q?=E6=AD=A2(=E9=9D=9E=E6=9A=B4=E5=8A=9B=E5=85=B3=E9=97=AD).?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- extractor/openrec/openrec_extractor.go | 215 +++++++++++++++++++++++ extractor/openrec/openrec_test.go | 226 ++++++------------------- extractor_field.go | 189 ++++++++++----------- go.mod | 1 - go.sum | 4 - sql/intimate_extractor.sql | 10 +- store.go | 38 +---- 7 files changed, 361 insertions(+), 322 deletions(-) create mode 100644 extractor/openrec/openrec_extractor.go diff --git a/extractor/openrec/openrec_extractor.go b/extractor/openrec/openrec_extractor.go new file mode 100644 index 0000000..7e846ba --- /dev/null +++ b/extractor/openrec/openrec_extractor.go @@ -0,0 +1,215 @@ +package main + +import ( + "database/sql" + "encoding/json" + "intimate" + "log" + "regexp" + "strconv" + "strings" + "time" + + "github.com/tidwall/gjson" +) + +// OpenrecExtractor 提取方法 +type OpenrecExtractor struct { + user *intimate.ExtractorSource + userLive *intimate.ExtractorSource + supporters *intimate.ExtractorSource +} + +func (oe *OpenrecExtractor) extractFollowers(clog intimate.ISet) { + extractor := oe.user.GetExtractor() + xp, err := extractor.XPathResult("//p[@class='c-global__user__count__row__right js-userCountFollowers']/text()") + if err != nil { + log.Println(err) + } + if !xp.NodeIter().Next() { + log.Println("不存在粉丝数") + } + + followers := strings.ReplaceAll(xp.String(), ",", "") + followersInt, err := strconv.ParseInt(followers, 10, 64) + if err != nil { + log.Println(err) + } + + clog.Set("Followers", sql.NullInt64{Int64: followersInt, Valid: true}) +} + +func (oe *OpenrecExtractor) extractAnchorName(ai intimate.ISet) { + extractor := oe.user.GetExtractor() + xp, err := extractor.XPathResult("//p[@class='c-global__user__profile__list__name__text official-icon--after']/text()") + if xp.NodeIter().Next() { + anchorName := xp.String() + ai.Set("AnchorName", anchorName) + } else { + log.Println(err) + } +} + +func (oe *OpenrecExtractor) extractViewsAndLiveStreaming(clog intimate.ISet) { + extractor := oe.user.GetExtractor() + // c-contents + xp, err := extractor.XPathResult("//ul[@class='c-contents']//p[@class='c-thumbnailVideo__footer__liveCount']/text()") + if err != nil { + log.Println(err) + } + if xp.NodeIter().Next() { + views := regexp.MustCompile(`[0-9,]+`).FindString(xp.String()) + views = strings.ReplaceAll(views, ",", "") + viewsint, err := strconv.Atoi(views) + if err != nil { + log.Println(err) + } + + clog.Set("Views", sql.NullInt64{Int64: int64(viewsint), Valid: true}) + clog.Set("IsLiveStreaming", int32(1)) + } +} + +func (oe *OpenrecExtractor) extractGiversAndGratuity(clog intimate.ISet) { + // extractor := oe.user.GetExtractor() + giverjson := oe.supporters.GetSource() + var givers []interface{} + var gratuity int64 = 0 + + for _, v := range giverjson.Array() { + giverSource := gjson.Parse(v.String()) + for _, item := range giverSource.Get("data.items").Array() { + givers = append(givers, item.Map()) + gratuity += item.Get("total_yells").Int() + } + } + + giversbytes, err := json.Marshal(givers) + if err != nil { + log.Println(err) + clog.Set("ErrorMsg", sql.NullString{String: err.Error(), Valid: true}) + } else { + clog.Set("Giver", giversbytes) + } + + clog.Set("Gratuity", sql.NullInt64{Int64: gratuity, Valid: true}) +} + +func (oe *OpenrecExtractor) extractLive(clog intimate.ISet) { + extractor := oe.userLive.GetExtractor() + mathes := regexp.MustCompile("MovieTitle__Title[^>]+>(.{1,50})").FindStringSubmatch(oe.userLive.GetSource().Str) + if len(mathes) == 2 { + + clog.Set("LiveTitle", sql.NullString{String: mathes[1], Valid: true}) + + content, err := extractor.XPathResult("//meta[@itemprop='uploadDate']/@content") + if err != nil { + log.Println(err) + } + + iter := content.NodeIter() + if iter.Next() { + tm, err := time.ParseInLocation("2006-01-02T15:04:05Z07:00", iter.Node().NodeValue(), time.Local) + if err != nil { + log.Println(err) + } + clog.Set("LiveStartTime", sql.NullTime{Time: tm.Local(), Valid: true}) + + duration, err := extractor.XPathResult("//meta[@itemprop='duration']/@content") + if err != nil { + log.Println(err) + } + + diter := duration.NodeIter() + if diter.Next() { + + dt, err := intimate.ParseDuration(diter.Node().NodeValue()) + if err != nil { + log.Println(err) + } + endtm := tm.Add(dt) + clog.Set("LiveEndTime", sql.NullTime{Time: endtm.Local(), Valid: true}) + } + } + } +} + +func (oe *OpenrecExtractor) extractTags(clog intimate.ISet) { + var tags []string + matheslist := regexp.MustCompile(`TagButton__Button[^>]+>(.{1,100})]+>(.{1,50})").FindStringSubmatch(livejson.Str) - if len(mathes) == 2 { - - clog.SetShowTitle(sql.NullString{String: mathes[1], Valid: true}) - - content, err := extractor.XPathResult("//meta[@itemprop='uploadDate']/@content") - if err != nil { - t.Error(err) - } - - iter := content.NodeIter() - if iter.Next() { - tm, err := time.ParseInLocation("2006-01-02T15:04:05Z07:00", iter.Node().NodeValue(), time.Local) - if err != nil { - t.Error(err) - } - clog.SetShowStartTime(sql.NullTime{Time: tm.Local(), Valid: true}) - - duration, err := extractor.XPathResult("//meta[@itemprop='duration']/@content") - if err != nil { - t.Error(err) - } - - diter := duration.NodeIter() - if diter.Next() { - - dt, err := intimate.ParseDuration(diter.Node().NodeValue()) - if err != nil { - log.Println(err) - } - endtm := tm.Add(dt) - clog.SetShowEndTime(sql.NullTime{Time: endtm.Local(), Valid: true}) - } - } - } - - var tags []string - matheslist := regexp.MustCompile(`TagButton__Button[^>]+>(.{1,100}) Date: Thu, 16 Jul 2020 16:22:14 +0800 Subject: [PATCH 13/14] fix: ParseDuration Method add: store UpdateOperator --- extractor/openrec/.gitignore | 3 ++- extractor/openrec/main.go | 3 ++- extractor/openrec/openrec_extractor.go | 18 ++++++++++++++---- extractor/openrec/openrec_test.go | 14 ++++++++++++-- store.go | 8 ++++++++ utils.go | 18 +++++++++++++++++- 6 files changed, 55 insertions(+), 9 deletions(-) diff --git a/extractor/openrec/.gitignore b/extractor/openrec/.gitignore index 589ccd4..f09112e 100644 --- a/extractor/openrec/.gitignore +++ b/extractor/openrec/.gitignore @@ -1,2 +1,3 @@ *.html -screenlog.* \ No newline at end of file +screenlog.* +openrec \ No newline at end of file diff --git a/extractor/openrec/main.go b/extractor/openrec/main.go index 86abea9..c0deaf0 100644 --- a/extractor/openrec/main.go +++ b/extractor/openrec/main.go @@ -11,5 +11,6 @@ package main */ func main() { - + oe := &OpenrecExtractor{} + oe.Execute() } diff --git a/extractor/openrec/openrec_extractor.go b/extractor/openrec/openrec_extractor.go index 7e846ba..de14fe8 100644 --- a/extractor/openrec/openrec_extractor.go +++ b/extractor/openrec/openrec_extractor.go @@ -136,7 +136,7 @@ func (oe *OpenrecExtractor) extractLive(clog intimate.ISet) { func (oe *OpenrecExtractor) extractTags(clog intimate.ISet) { var tags []string - matheslist := regexp.MustCompile(`TagButton__Button[^>]+>(.{1,100})]+>(.{1,100})`).FindAllStringSubmatch(oe.userLive.GetSource().Str, -1) for _, m := range matheslist { tags = append(tags, m[1]) } @@ -150,16 +150,19 @@ func (oe *OpenrecExtractor) extractTags(clog intimate.ISet) { } func (oe *OpenrecExtractor) Execute() { + collect := intimate.NewExtractorStore() store := intimate.NewSourceStore("source_openrec") for { source, err := store.Pop(string(intimate.TTOpenrecRanking), 100) + if err != nil { log.Println(err) return } + source.SetOperator(int32(intimate.OperatorError)) anchorId := source.GetSource().String ai := &intimate.AnchorInfo{} @@ -167,12 +170,10 @@ func (oe *OpenrecExtractor) Execute() { ai.SetPlatform(string(intimate.Popenrec)) sdata := source.GetExt().([]byte) - if gjson.ValidBytes(sdata) { result := gjson.ParseBytes(sdata) datamap := result.Map() - oe := &OpenrecExtractor{} oe.user = intimate.NewExtractorSource(datamap["user"]) oe.user.CreateExtractor() @@ -198,6 +199,8 @@ func (oe *OpenrecExtractor) Execute() { Uid, err := collect.InsertAnchorInfo(ai) if err != nil { log.Println(err) + source.SetErrorMsg(sql.NullString{String: err.Error(), Valid: true}) + store.UpdateOperator(source) return } @@ -206,7 +209,14 @@ func (oe *OpenrecExtractor) Execute() { clog.Set("AnchorId", anchorId) clog.Set("UpdateTime", source.GetUpdateTime()) - collect.InsertCollectLog(clog) + if err = collect.InsertCollectLog(clog); err != nil { + source.SetErrorMsg(sql.NullString{String: err.Error(), Valid: true}) + store.UpdateOperator(source) + return + } + + source.SetOperator(int32(intimate.OperatorExtractorOK)) + store.UpdateOperator(source) } else { log.Println("data is not json:\n", string(sdata)) } diff --git a/extractor/openrec/openrec_test.go b/extractor/openrec/openrec_test.go index 0480a65..8190607 100644 --- a/extractor/openrec/openrec_test.go +++ b/extractor/openrec/openrec_test.go @@ -100,11 +100,13 @@ func TestExtractor(t *testing.T) { for { source, err := store.Pop(string(intimate.TTOpenrecRanking), 100) + if err != nil { log.Println(err) return } + source.SetOperator(int32(intimate.OperatorError)) anchorId := source.GetSource().String ai := &intimate.AnchorInfo{} @@ -112,7 +114,6 @@ func TestExtractor(t *testing.T) { ai.SetPlatform(string(intimate.Popenrec)) sdata := source.GetExt().([]byte) - if gjson.ValidBytes(sdata) { result := gjson.ParseBytes(sdata) datamap := result.Map() @@ -143,6 +144,8 @@ func TestExtractor(t *testing.T) { Uid, err := collect.InsertAnchorInfo(ai) if err != nil { t.Error(err) + source.SetErrorMsg(sql.NullString{String: err.Error(), Valid: true}) + store.UpdateOperator(source) return } @@ -151,7 +154,14 @@ func TestExtractor(t *testing.T) { clog.Set("AnchorId", anchorId) clog.Set("UpdateTime", source.GetUpdateTime()) - collect.InsertCollectLog(clog) + if err = collect.InsertCollectLog(clog); err != nil { + source.SetErrorMsg(sql.NullString{String: err.Error(), Valid: true}) + store.UpdateOperator(source) + return + } + + source.SetOperator(int32(intimate.OperatorExtractorOK)) + store.UpdateOperator(source) } else { t.Error("data is not json:\n", string(sdata)) } diff --git a/store.go b/store.go index 9b96fdf..900975f 100644 --- a/store.go +++ b/store.go @@ -13,6 +13,8 @@ type OperatorFlag int32 const ( // OperatorOK 等待被处理 OperatorOK OperatorFlag = 100 + // OperatorExtractorOK 提取数据完成 + OperatorExtractorOK OperatorFlag = 200 // OperatorWait 等待被处理 OperatorWait OperatorFlag = 1000 // OperatorError 错误标志 @@ -67,6 +69,12 @@ func (store *SourceStore) Update(isource IUpdateSource) { store.errorAlarm(err) } +// UpdateOperator 更新数据操作标志位 +func (store *SourceStore) UpdateOperator(isource IUpdateSource) { + _, err := store.db.Exec("update "+store.table+" set operator = ?, error_msg = ? where uid = ?", isource.GetOperator(), isource.GetErrorMsg(), isource.GetUid()) + store.errorAlarm(err) +} + // UpdateError 更新错误数据 func (store *SourceStore) UpdateError(isource IUpdateSource, err error) { isource.SetOperator(int32(OperatorError)) diff --git a/utils.go b/utils.go index f922bf5..9df8fbb 100644 --- a/utils.go +++ b/utils.go @@ -19,7 +19,23 @@ func init() { // ParseDuration time to duration eg: 1:40:00 -> time.Duration func ParseDuration(dt string) (time.Duration, error) { - tdt, err := time.Parse("15:04:05", dt) + + var parse []byte = []byte("00:00:00") + + j := len(parse) - 1 + for i := len(dt) - 1; i >= 0; i-- { + c := dt[i] + if c != ':' { + parse[j] = dt[i] + } else { + for parse[j] != ':' { + j-- + } + } + j-- + } + + tdt, err := time.Parse("15:04:05", string(parse)) if err != nil { return time.Duration(0), err From 93efe36d0b624e002e3aa0f623e0608ae1639f63 Mon Sep 17 00:00:00 2001 From: eson Date: Thu, 16 Jul 2020 18:31:13 +0800 Subject: [PATCH 14/14] =?UTF-8?q?add:=20=E4=BC=98=E9=9B=85=E5=81=9C?= =?UTF-8?q?=E6=9C=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- extractor/openrec/.gitignore | 3 - extractor/openrec/openrec_test.go | 170 ----------------- extractor/openrec_extractor/.gitignore | 4 + .../{openrec => openrec_extractor}/main.go | 0 .../openrec_extractor.go | 171 ++++++++++-------- extractor/openrec_extractor/openrec_test.go | 96 ++++++++++ store.go | 4 +- tasks/openrec/openrec_task1/.gitignore | 3 +- tasks/openrec/openrec_task1/task_openrec.go | 15 +- tasks/openrec/openrec_task2/task_openrec.go | 15 +- 10 files changed, 226 insertions(+), 255 deletions(-) delete mode 100644 extractor/openrec/.gitignore delete mode 100644 extractor/openrec/openrec_test.go create mode 100644 extractor/openrec_extractor/.gitignore rename extractor/{openrec => openrec_extractor}/main.go (100%) rename extractor/{openrec => openrec_extractor}/openrec_extractor.go (90%) create mode 100644 extractor/openrec_extractor/openrec_test.go diff --git a/extractor/openrec/.gitignore b/extractor/openrec/.gitignore deleted file mode 100644 index f09112e..0000000 --- a/extractor/openrec/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -*.html -screenlog.* -openrec \ No newline at end of file diff --git a/extractor/openrec/openrec_test.go b/extractor/openrec/openrec_test.go deleted file mode 100644 index 8190607..0000000 --- a/extractor/openrec/openrec_test.go +++ /dev/null @@ -1,170 +0,0 @@ -package main - -import ( - "database/sql" - "intimate" - "io/ioutil" - "log" - "os" - "regexp" - "testing" - "time" - - "github.com/lestrrat-go/libxml2" - "github.com/tidwall/gjson" -) - -func TestCase0(t *testing.T) { - f, err := os.Open("./test.html") - if err != nil { - panic(err) - } - data, err := ioutil.ReadAll(f) - if err != nil { - panic(err) - } - - matheslist := regexp.MustCompile(`TagButton__Button[^>]+>(.{1,100})" - - xresult, err := doc.Find("/html/head") - ele, err := doc.CreateElement(`META`) - - if err != nil { - panic(err) - } - ele.SetAttribute("charset", "utf-8") - - if err != nil { - panic(err) - } - - iter := xresult.NodeIter() - if iter.Next() { - n := iter.Node() - - err = n.AddChild(ele) - // childs, err := n.ChildNodes() - if err != nil { - t.Error(err) - } - t.Error(n) - } - - xr, err := doc.Find("//h1[ contains(@class, 'MovieTitle__Title')]") - if err != nil { - panic(nil) - } - - t.Error(xr) -} - -func TestExtractor(t *testing.T) { - collect := intimate.NewExtractorStore() - store := intimate.NewSourceStore("source_openrec") - - for { - source, err := store.Pop(string(intimate.TTOpenrecRanking), 100) - - if err != nil { - log.Println(err) - return - } - - source.SetOperator(int32(intimate.OperatorError)) - anchorId := source.GetSource().String - - ai := &intimate.AnchorInfo{} - ai.SetAnchorId(anchorId) - ai.SetPlatform(string(intimate.Popenrec)) - - sdata := source.GetExt().([]byte) - if gjson.ValidBytes(sdata) { - result := gjson.ParseBytes(sdata) - datamap := result.Map() - - oe := &OpenrecExtractor{} - oe.user = intimate.NewExtractorSource(datamap["user"]) - oe.user.CreateExtractor() - - oe.userLive = intimate.NewExtractorSource(datamap["user_live"]) - oe.userLive.CreateExtractor() - - oe.supporters = intimate.NewExtractorSource(datamap["supporters"]) - - clog := &intimate.CollectLog{} - - oe.extractFollowers(clog) - oe.extractAnchorName(ai) - oe.extractViewsAndLiveStreaming(clog) - oe.extractGiversAndGratuity(clog) - oe.extractLive(clog) - oe.extractTags(clog) - - ai.Set("UpdateTime", source.GetUpdateTime()) - - LiveUrl := "https://www.openrec.tv/live/" + anchorId - ai.Set("LiveUrl", sql.NullString{String: LiveUrl, Valid: true}) - - Uid, err := collect.InsertAnchorInfo(ai) - if err != nil { - t.Error(err) - source.SetErrorMsg(sql.NullString{String: err.Error(), Valid: true}) - store.UpdateOperator(source) - return - } - - clog.Set("Uid", Uid) - clog.Set("Platform", string(intimate.Popenrec)) - clog.Set("AnchorId", anchorId) - clog.Set("UpdateTime", source.GetUpdateTime()) - - if err = collect.InsertCollectLog(clog); err != nil { - source.SetErrorMsg(sql.NullString{String: err.Error(), Valid: true}) - store.UpdateOperator(source) - return - } - - source.SetOperator(int32(intimate.OperatorExtractorOK)) - store.UpdateOperator(source) - } else { - t.Error("data is not json:\n", string(sdata)) - } - } - -} diff --git a/extractor/openrec_extractor/.gitignore b/extractor/openrec_extractor/.gitignore new file mode 100644 index 0000000..89d3e55 --- /dev/null +++ b/extractor/openrec_extractor/.gitignore @@ -0,0 +1,4 @@ +*.html +log +screenlog.* +openrec_extractor \ No newline at end of file diff --git a/extractor/openrec/main.go b/extractor/openrec_extractor/main.go similarity index 100% rename from extractor/openrec/main.go rename to extractor/openrec_extractor/main.go diff --git a/extractor/openrec/openrec_extractor.go b/extractor/openrec_extractor/openrec_extractor.go similarity index 90% rename from extractor/openrec/openrec_extractor.go rename to extractor/openrec_extractor/openrec_extractor.go index de14fe8..87c4ee5 100644 --- a/extractor/openrec/openrec_extractor.go +++ b/extractor/openrec_extractor/openrec_extractor.go @@ -5,9 +5,13 @@ import ( "encoding/json" "intimate" "log" + "os" + "os/signal" "regexp" "strconv" "strings" + "sync/atomic" + "syscall" "time" "github.com/tidwall/gjson" @@ -20,6 +24,96 @@ type OpenrecExtractor struct { supporters *intimate.ExtractorSource } +func (oe *OpenrecExtractor) Execute() { + + var loop int32 = 1 + + go func() { + signalchan := make(chan os.Signal) + signal.Notify(signalchan, syscall.SIGKILL, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGSTOP) + log.Println("accept stop command:", <-signalchan) + atomic.StoreInt32(&loop, 0) + }() + + collect := intimate.NewExtractorStore() + store := intimate.NewSourceStore("source_openrec") + var lasterr error = nil + + for atomic.LoadInt32(&loop) > 0 { + + source, err := store.Pop(string(intimate.TTOpenrecRanking), 100) + if err != nil { + if err != lasterr { + log.Println(err, lasterr) + lasterr = err + } + time.Sleep(time.Second * 2) + continue + } + + source.SetOperator(int32(intimate.OperatorError)) + anchorId := source.GetSource().String + + ai := &intimate.AnchorInfo{} + ai.SetAnchorId(anchorId) + ai.SetPlatform(string(intimate.Popenrec)) + + sdata := source.GetExt().([]byte) + if gjson.ValidBytes(sdata) { + result := gjson.ParseBytes(sdata) + datamap := result.Map() + + oe.user = intimate.NewExtractorSource(datamap["user"]) + oe.user.CreateExtractor() + + oe.userLive = intimate.NewExtractorSource(datamap["user_live"]) + oe.userLive.CreateExtractor() + + oe.supporters = intimate.NewExtractorSource(datamap["supporters"]) + + clog := &intimate.CollectLog{} + + log.Println(anchorId) + + oe.extractFollowers(clog) + oe.extractAnchorName(ai) + oe.extractViewsAndLiveStreaming(clog) + oe.extractGiversAndGratuity(clog) + oe.extractLive(clog) + oe.extractTags(clog) + + ai.Set("UpdateTime", source.GetUpdateTime()) + + LiveUrl := "https://www.openrec.tv/live/" + anchorId + ai.Set("LiveUrl", sql.NullString{String: LiveUrl, Valid: true}) + + Uid, err := collect.InsertAnchorInfo(ai) + if err != nil { + log.Println(err) + source.SetErrorMsg(sql.NullString{String: err.Error(), Valid: true}) + store.UpdateOperator(source) + return + } + + clog.Set("Uid", Uid) + clog.Set("Platform", string(intimate.Popenrec)) + clog.Set("AnchorId", anchorId) + clog.Set("UpdateTime", source.GetUpdateTime()) + + if err = collect.InsertCollectLog(clog); err != nil { + source.SetErrorMsg(sql.NullString{String: err.Error(), Valid: true}) + store.UpdateOperator(source) + return + } + + source.SetOperator(int32(intimate.OperatorExtractorOK)) + store.UpdateOperator(source) + } else { + log.Println("data is not json:\n", string(sdata)) + } + } +} + func (oe *OpenrecExtractor) extractFollowers(clog intimate.ISet) { extractor := oe.user.GetExtractor() xp, err := extractor.XPathResult("//p[@class='c-global__user__count__row__right js-userCountFollowers']/text()") @@ -136,7 +230,7 @@ func (oe *OpenrecExtractor) extractLive(clog intimate.ISet) { func (oe *OpenrecExtractor) extractTags(clog intimate.ISet) { var tags []string - matheslist := regexp.MustCompile(`]+>(.{1,100})`).FindAllStringSubmatch(oe.userLive.GetSource().Str, -1) + matheslist := regexp.MustCompile(`<[^>]+TagButton[^>]+>([^<]{1,100})<`).FindAllStringSubmatch(oe.userLive.GetSource().Str, -1) for _, m := range matheslist { tags = append(tags, m[1]) } @@ -148,78 +242,3 @@ func (oe *OpenrecExtractor) extractTags(clog intimate.ISet) { clog.Set("Tags", tagsBytes) } - -func (oe *OpenrecExtractor) Execute() { - - collect := intimate.NewExtractorStore() - store := intimate.NewSourceStore("source_openrec") - - for { - source, err := store.Pop(string(intimate.TTOpenrecRanking), 100) - - if err != nil { - log.Println(err) - return - } - - source.SetOperator(int32(intimate.OperatorError)) - anchorId := source.GetSource().String - - ai := &intimate.AnchorInfo{} - ai.SetAnchorId(anchorId) - ai.SetPlatform(string(intimate.Popenrec)) - - sdata := source.GetExt().([]byte) - if gjson.ValidBytes(sdata) { - result := gjson.ParseBytes(sdata) - datamap := result.Map() - - oe.user = intimate.NewExtractorSource(datamap["user"]) - oe.user.CreateExtractor() - - oe.userLive = intimate.NewExtractorSource(datamap["user_live"]) - oe.userLive.CreateExtractor() - - oe.supporters = intimate.NewExtractorSource(datamap["supporters"]) - - clog := &intimate.CollectLog{} - - oe.extractFollowers(clog) - oe.extractAnchorName(ai) - oe.extractViewsAndLiveStreaming(clog) - oe.extractGiversAndGratuity(clog) - oe.extractLive(clog) - oe.extractTags(clog) - - ai.Set("UpdateTime", source.GetUpdateTime()) - - LiveUrl := "https://www.openrec.tv/live/" + anchorId - ai.Set("LiveUrl", sql.NullString{String: LiveUrl, Valid: true}) - - Uid, err := collect.InsertAnchorInfo(ai) - if err != nil { - log.Println(err) - source.SetErrorMsg(sql.NullString{String: err.Error(), Valid: true}) - store.UpdateOperator(source) - return - } - - clog.Set("Uid", Uid) - clog.Set("Platform", string(intimate.Popenrec)) - clog.Set("AnchorId", anchorId) - clog.Set("UpdateTime", source.GetUpdateTime()) - - if err = collect.InsertCollectLog(clog); err != nil { - source.SetErrorMsg(sql.NullString{String: err.Error(), Valid: true}) - store.UpdateOperator(source) - return - } - - source.SetOperator(int32(intimate.OperatorExtractorOK)) - store.UpdateOperator(source) - } else { - log.Println("data is not json:\n", string(sdata)) - } - } - -} diff --git a/extractor/openrec_extractor/openrec_test.go b/extractor/openrec_extractor/openrec_test.go new file mode 100644 index 0000000..5212f04 --- /dev/null +++ b/extractor/openrec_extractor/openrec_test.go @@ -0,0 +1,96 @@ +package main + +import ( + "io/ioutil" + "os" + "regexp" + "testing" + "time" + + "github.com/lestrrat-go/libxml2" +) + +func TestCase0(t *testing.T) { + f, err := os.Open("./test.html") + if err != nil { + panic(err) + } + data, err := ioutil.ReadAll(f) + if err != nil { + panic(err) + } + + matheslist := regexp.MustCompile(`TagButton__Button[^>]+>(.{1,100})" + + xresult, err := doc.Find("/html/head") + ele, err := doc.CreateElement(`META`) + + if err != nil { + panic(err) + } + ele.SetAttribute("charset", "utf-8") + + if err != nil { + panic(err) + } + + iter := xresult.NodeIter() + if iter.Next() { + n := iter.Node() + + err = n.AddChild(ele) + // childs, err := n.ChildNodes() + if err != nil { + t.Error(err) + } + t.Error(n) + } + + xr, err := doc.Find("//h1[ contains(@class, 'MovieTitle__Title')]") + if err != nil { + panic(nil) + } + + t.Error(xr) +} + +func TestExtractor(t *testing.T) { + oe := &OpenrecExtractor{} + oe.Execute() +} diff --git a/store.go b/store.go index 900975f..56d9cab 100644 --- a/store.go +++ b/store.go @@ -94,7 +94,6 @@ func (store *SourceStore) Pop(targetType string, operators ...int32) (IUpdateSou tx, err := store.db.Begin() if err != nil { - log.Println(err, targetType) return nil, err } var args = []interface{}{targetType} @@ -127,7 +126,6 @@ func (store *SourceStore) Pop(targetType string, operators ...int32) (IUpdateSou // uid, url, target_type, source, ext, operator err = row.Scan(&s.Uid, &s.Url, &s.TargetType, &s.Source, &s.Ext, &s.Operator, &s.UpdateTime) if err != nil { - log.Println(err, targetType) return nil, err } s.SetLastOperator(s.Operator) @@ -190,7 +188,7 @@ func (store *ExtractorStore) InsertAnchorInfo(isource IGetAnchorInfo) (Uid int64 log.Println(err) return 0, err } - log.Println(isource.GetPlatform(), isource.GetAnchorId()) + row := tx.QueryRow(selectSQL+` limit 1 for update`, isource.GetPlatform(), isource.GetAnchorId()) var uid int64 diff --git a/tasks/openrec/openrec_task1/.gitignore b/tasks/openrec/openrec_task1/.gitignore index 42d0e6c..adfb476 100644 --- a/tasks/openrec/openrec_task1/.gitignore +++ b/tasks/openrec/openrec_task1/.gitignore @@ -1 +1,2 @@ -openrec_task1 \ No newline at end of file +openrec_task1 +log \ No newline at end of file diff --git a/tasks/openrec/openrec_task1/task_openrec.go b/tasks/openrec/openrec_task1/task_openrec.go index 7976b96..d59fd41 100644 --- a/tasks/openrec/openrec_task1/task_openrec.go +++ b/tasks/openrec/openrec_task1/task_openrec.go @@ -4,7 +4,11 @@ import ( "database/sql" "intimate" "log" + "os" + "os/signal" "strconv" + "sync/atomic" + "syscall" "time" "github.com/474420502/hunter" @@ -42,7 +46,16 @@ type OpenrecRanking struct { // Execute 执行方法 func (or *OpenrecRanking) Execute(cxt *hunter.TaskContext) { - for { + var loop int32 = 1 + + go func() { + signalchan := make(chan os.Signal) + signal.Notify(signalchan, syscall.SIGKILL, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGSTOP) + log.Println("accept stop command:", <-signalchan) + atomic.StoreInt32(&loop, 0) + }() + + for atomic.LoadInt32(&loop) > 0 { resp, err := cxt.Hunt() if err != nil { diff --git a/tasks/openrec/openrec_task2/task_openrec.go b/tasks/openrec/openrec_task2/task_openrec.go index 286914c..23c570e 100644 --- a/tasks/openrec/openrec_task2/task_openrec.go +++ b/tasks/openrec/openrec_task2/task_openrec.go @@ -5,7 +5,11 @@ import ( "encoding/json" "intimate" "log" + "os" + "os/signal" "strconv" + "sync/atomic" + "syscall" "time" "github.com/474420502/gcurl" @@ -31,7 +35,16 @@ type OpenrecExtratorRanking struct { // Execute 执行方法 func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) { - for { + var loop int32 = 1 + + go func() { + signalchan := make(chan os.Signal) + signal.Notify(signalchan, syscall.SIGKILL, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGSTOP) + log.Println("accept stop command:", <-signalchan) + atomic.StoreInt32(&loop, 0) + }() + + for atomic.LoadInt32(&loop) > 0 { source, err := store.Pop(string(intimate.TTOpenrecUser))