From d91e6be26961b05b55b5f45c3931ad6b6363fbaf Mon Sep 17 00:00:00 2001 From: eson Date: Wed, 12 Aug 2020 17:42:27 +0800 Subject: [PATCH 1/8] rename api name xml -> html --- .../twitcasting_extractor/twitcasting_extractor.go | 7 +++++-- .../twitcasting_extractor_test.go | 2 +- go.mod | 2 +- go.sum | 10 ++++------ tasks/twitcasting/twitcasting_task1/twitcasting.go | 2 +- 5 files changed, 12 insertions(+), 11 deletions(-) diff --git a/extractor/twitcasting_extractor/twitcasting_extractor.go b/extractor/twitcasting_extractor/twitcasting_extractor.go index 0550e6a..6351c3d 100644 --- a/extractor/twitcasting_extractor/twitcasting_extractor.go +++ b/extractor/twitcasting_extractor/twitcasting_extractor.go @@ -49,7 +49,7 @@ func main() { continue } var ldata *LiveData - etor := extractor.ExtractXml(resp.Content()) + etor := extractor.ExtractHtml(resp.Content()) ldata = etor.GetObjectByTag(LiveData{}).(*LiveData) ldata.MaxViews = regexp.MustCompile("\\d+").FindString(ldata.MaxViews) coincount := 0 @@ -59,11 +59,14 @@ func main() { giverurl := streamer.LiveUrl.String + "/backers/" + strconv.Itoa(i) resp, err = ses.Get(giverurl).Execute() if err != nil { + estore.UpdateError(streamer, err) log.Panic(err) } - etor := extractor.ExtractXml(resp.Content()) + + etor := extractor.ExtractHtml(resp.Content()) xp, err := etor.XPaths("//td[@class='tw-memorial-table-recent-point']") if err != nil { + estore.UpdateError(streamer, err) log.Panic(err) } diff --git a/extractor/twitcasting_extractor/twitcasting_extractor_test.go b/extractor/twitcasting_extractor/twitcasting_extractor_test.go index 811b2d3..3ed48ff 100644 --- a/extractor/twitcasting_extractor/twitcasting_extractor_test.go +++ b/extractor/twitcasting_extractor/twitcasting_extractor_test.go @@ -10,7 +10,7 @@ import ( // Follower string `exp:".//span[@class='tw-user-nav-list-count']" method:"Text"` // } -func TestMain(t *testing.T) { +func estMain(t *testing.T) { main() } diff --git a/go.mod b/go.mod index 35e0665..4f1e83b 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module intimate go 1.14 require ( - github.com/474420502/extractor v0.5.2 + github.com/474420502/extractor v0.7.1 github.com/474420502/focus v0.12.0 github.com/474420502/gcurl v0.1.2 github.com/474420502/hunter v0.3.4 diff --git a/go.sum b/go.sum index 32dceb6..3e96929 100644 --- a/go.sum +++ b/go.sum @@ -2,16 +2,14 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.41.0/go.mod h1:OauMR7DV8fzvZIl2qg6rkaIhD/vmgk4iwEw/h6ercmg= -github.com/474420502/extractor v0.5.1 h1:A1heJJSYbV9nEaUHfl3/1HYXcsBQfsTzAHikgwg2IF0= -github.com/474420502/extractor v0.5.1/go.mod h1:vkqsbi7wXPqyi5Q5dchcGjiaWHbgOJOAEcwonBiAs/E= -github.com/474420502/extractor v0.5.2 h1:ndgrAkxJjQg0Nrbq3AX2/xAnmIJNxSHRFGQ78wEtWj4= -github.com/474420502/extractor v0.5.2/go.mod h1:vkqsbi7wXPqyi5Q5dchcGjiaWHbgOJOAEcwonBiAs/E= +github.com/474420502/extractor v0.7.1 h1:Tdbi5s8wfyw8B3CybW0j/vnbBnrBDM0Q0lJl4AYv0fg= +github.com/474420502/extractor v0.7.1/go.mod h1:92J6QZKstpAKGhv+DibemhQbR/d6lJ+ftyR/ZHmeJ0w= github.com/474420502/focus v0.12.0 h1:+icbmj7IEOefvTegHt5EpcHt6WFbe2miIrceUJx2Evo= github.com/474420502/focus v0.12.0/go.mod h1:d0PMjtMxFz1a9HIhwyFPkWa+JF+0LgOrEUfd8iZka6s= github.com/474420502/gcurl v0.1.2 h1:ON9Yz3IgAdtDlFlHfkAJ3aIEBDxH0RiViPE5ST5ohKg= github.com/474420502/gcurl v0.1.2/go.mod h1:hws5q/Ao64bXLLDnldz9VyTQUndTWc/i5DzdEazFfoM= -github.com/474420502/htmlquery v1.2.4-0.20200810165859-a0e2c521c7c2 h1:4F1tpJ+sEkb3N+XD+Wb9MFiQmOMm3bHp8QUP+BQvkVk= -github.com/474420502/htmlquery v1.2.4-0.20200810165859-a0e2c521c7c2/go.mod h1:AoSN890esHwNKecV0tCs+W0ele1xgFL1Jqk6UcrdxgU= +github.com/474420502/htmlquery v1.2.4-0.20200812072201-e871dd09247a h1:E1T6CYQKsUn7fMvNbeKfISjBLfOJjZX4KpWwStT20Kc= +github.com/474420502/htmlquery v1.2.4-0.20200812072201-e871dd09247a/go.mod h1:AoSN890esHwNKecV0tCs+W0ele1xgFL1Jqk6UcrdxgU= github.com/474420502/hunter v0.3.4 h1:fyLAgI84jWe3IcqsISC53j1w3CXI1FERxX//Potns0M= github.com/474420502/hunter v0.3.4/go.mod h1:pe4Xr/I+2agvq339vS/OZV+EiHAWtpXQs75rioSW9oA= github.com/474420502/requests v1.6.0 h1:f4h4j40eT0P5whhg9LdkotD8CaKjtuDu/vz9iSUkCgY= diff --git a/tasks/twitcasting/twitcasting_task1/twitcasting.go b/tasks/twitcasting/twitcasting_task1/twitcasting.go index 58e2cba..89204c2 100644 --- a/tasks/twitcasting/twitcasting_task1/twitcasting.go +++ b/tasks/twitcasting/twitcasting_task1/twitcasting.go @@ -58,7 +58,7 @@ func Execute() { // log.Panic(err) } - etor := extractor.ExtractXml(resp.Content()) + etor := extractor.ExtractHtml(resp.Content()) result, err := etor.XPaths("//p[@class='taglist']/a[contains(@class, 'tag')]/@href") if err != nil { panic(err) From 36c28826baab13b50ae8f4a4ea15af69a479005d Mon Sep 17 00:00:00 2001 From: eson Date: Wed, 12 Aug 2020 18:18:25 +0800 Subject: [PATCH 2/8] change some api --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 4f1e83b..fd195ef 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module intimate go 1.14 require ( - github.com/474420502/extractor v0.7.1 + github.com/474420502/extractor v0.7.2 github.com/474420502/focus v0.12.0 github.com/474420502/gcurl v0.1.2 github.com/474420502/hunter v0.3.4 diff --git a/go.sum b/go.sum index 3e96929..dac54a5 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.41.0/go.mod h1:OauMR7DV8fzvZIl2qg6rkaIhD/vmgk4iwEw/h6ercmg= -github.com/474420502/extractor v0.7.1 h1:Tdbi5s8wfyw8B3CybW0j/vnbBnrBDM0Q0lJl4AYv0fg= -github.com/474420502/extractor v0.7.1/go.mod h1:92J6QZKstpAKGhv+DibemhQbR/d6lJ+ftyR/ZHmeJ0w= +github.com/474420502/extractor v0.7.2 h1:idZnsekOKRV8fpJwsRcr6Ol7KSphKXe9tc+JJXgGqQ4= +github.com/474420502/extractor v0.7.2/go.mod h1:92J6QZKstpAKGhv+DibemhQbR/d6lJ+ftyR/ZHmeJ0w= github.com/474420502/focus v0.12.0 h1:+icbmj7IEOefvTegHt5EpcHt6WFbe2miIrceUJx2Evo= github.com/474420502/focus v0.12.0/go.mod h1:d0PMjtMxFz1a9HIhwyFPkWa+JF+0LgOrEUfd8iZka6s= github.com/474420502/gcurl v0.1.2 h1:ON9Yz3IgAdtDlFlHfkAJ3aIEBDxH0RiViPE5ST5ohKg= From b20f97f7c9bd4b1fb667d207bc42f7dcffd1efc7 Mon Sep 17 00:00:00 2001 From: eson Date: Thu, 13 Aug 2020 19:11:53 +0800 Subject: [PATCH 3/8] =?UTF-8?q?1.=20=E4=BF=AE=E5=A4=8D=20store=5Fextractor?= =?UTF-8?q?=20Update=20=E9=94=99=E8=AF=AF.=202.=20=E9=87=8D=E6=9E=84?= =?UTF-8?q?=E5=89=8D=E4=B8=A4=E4=B8=AA=E7=BD=91=E7=AB=99=E7=9A=84=E4=BB=A3?= =?UTF-8?q?=E7=A0=81=E6=9E=B6=E6=9E=84=E4=B8=8D=E5=90=88=E7=90=86.=203.=20?= =?UTF-8?q?=E5=87=8F=E5=B0=91=E5=86=97=E4=BD=99=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../openrec_extractor/openrec_extractor.go | 31 +++++---- go.mod | 4 +- go.sum | 4 ++ store.go | 7 +- tasks/openrec/openrec_task1/main.go | 5 +- tasks/openrec/openrec_task1/task_openrec.go | 62 ++++-------------- .../openrec_task1/task_openrec_test.go | 5 +- tasks/openrec/openrec_task2/main.go | 5 +- tasks/openrec/openrec_task2/task_openrec.go | 64 ++++++++----------- .../openrec_task2/task_openrec_test.go | 7 +- tasks/twitch/twitch_task1/main.go | 3 +- tasks/twitch/twitch_task1/task_twitch.go | 6 +- tasks/twitch/twitch_task1/task_twitch_test.go | 5 +- tasks/twitch/twitch_task2/main.go | 4 +- tasks/twitch/twitch_task2/task_twitch.go | 10 +-- tasks/twitch/twitch_task2/task_twitch_test.go | 2 +- 16 files changed, 76 insertions(+), 148 deletions(-) diff --git a/extractor/openrec_extractor/openrec_extractor.go b/extractor/openrec_extractor/openrec_extractor.go index 41cf76d..b8a8943 100644 --- a/extractor/openrec_extractor/openrec_extractor.go +++ b/extractor/openrec_extractor/openrec_extractor.go @@ -5,13 +5,9 @@ import ( "encoding/json" "intimate" "log" - "os" - "os/signal" "regexp" "strconv" "strings" - "sync/atomic" - "syscall" "time" "github.com/tidwall/gjson" @@ -29,17 +25,10 @@ type OpenrecExtractor struct { func (oe *OpenrecExtractor) Execute() { - var loop int32 = 1 - - go func() { - signalchan := make(chan os.Signal) - signal.Notify(signalchan, syscall.SIGKILL, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGSTOP) - log.Println("accept stop command:", <-signalchan) - atomic.StoreInt32(&loop, 0) - }() + ps := intimate.NewPerfectShutdown() var lasterr error = nil - for atomic.LoadInt32(&loop) > 0 { + for !ps.IsClose() { var err error source, err := sstore.Pop(intimate.TOpenrecUser, 0) @@ -60,7 +49,7 @@ func (oe *OpenrecExtractor) Execute() { streamer := &intimate.Streamer{} streamer.UserId = userId - streamer.Platform = intimate.Popenrec + // streamer.Platform = intimate.Popenrec htmlUser := datamap["html_user"] oe.user = intimate.NewExtractorSource(&htmlUser) @@ -94,10 +83,20 @@ func (oe *OpenrecExtractor) Execute() { logUid := estore.InsertClog(clog) LiveUrl := "https://www.openrec.tv/live/" + userId + streamer.LiveUrl = sql.NullString{String: LiveUrl, Valid: true} streamer.LatestLogUid = logUid - streamer.Operator = 0 - estore.UpdateStreamer(streamer) + // streamer.Operator = 0 + + log.Println(streamer.UserId) + estore.Update(streamer, + "user_name", streamer.UserName, + "user_id", streamer.UserId, + "live_url", streamer.LiveUrl, + "latest_log_uid", streamer.LatestLogUid, + "update_time", streamer.UpdateTime, + "tags", streamer.Tags, + ) source.Operator = int32(intimate.OperatorExtractorOK) sstore.UpdateOperator(source) diff --git a/go.mod b/go.mod index fd195ef..36800bf 100644 --- a/go.mod +++ b/go.mod @@ -5,9 +5,9 @@ go 1.14 require ( github.com/474420502/extractor v0.7.2 github.com/474420502/focus v0.12.0 - github.com/474420502/gcurl v0.1.2 + github.com/474420502/gcurl v0.2.0 github.com/474420502/hunter v0.3.4 - github.com/474420502/requests v1.6.0 + github.com/474420502/requests v1.7.0 github.com/go-sql-driver/mysql v1.5.0 github.com/lestrrat-go/libxml2 v0.0.0-20200215080510-6483566f52cb github.com/tebeka/selenium v0.9.9 diff --git a/go.sum b/go.sum index dac54a5..5ed6621 100644 --- a/go.sum +++ b/go.sum @@ -8,12 +8,16 @@ github.com/474420502/focus v0.12.0 h1:+icbmj7IEOefvTegHt5EpcHt6WFbe2miIrceUJx2Ev github.com/474420502/focus v0.12.0/go.mod h1:d0PMjtMxFz1a9HIhwyFPkWa+JF+0LgOrEUfd8iZka6s= github.com/474420502/gcurl v0.1.2 h1:ON9Yz3IgAdtDlFlHfkAJ3aIEBDxH0RiViPE5ST5ohKg= github.com/474420502/gcurl v0.1.2/go.mod h1:hws5q/Ao64bXLLDnldz9VyTQUndTWc/i5DzdEazFfoM= +github.com/474420502/gcurl v0.2.0 h1:m6+vw4NX4f5Tfp7c3nuaIgHUE/7zTX6K3xK+pTCBoCo= +github.com/474420502/gcurl v0.2.0/go.mod h1:kJZDbgXn5wbAaR+hhBi4Sbw44P4igJ7qYXC6mejLuhQ= github.com/474420502/htmlquery v1.2.4-0.20200812072201-e871dd09247a h1:E1T6CYQKsUn7fMvNbeKfISjBLfOJjZX4KpWwStT20Kc= github.com/474420502/htmlquery v1.2.4-0.20200812072201-e871dd09247a/go.mod h1:AoSN890esHwNKecV0tCs+W0ele1xgFL1Jqk6UcrdxgU= github.com/474420502/hunter v0.3.4 h1:fyLAgI84jWe3IcqsISC53j1w3CXI1FERxX//Potns0M= github.com/474420502/hunter v0.3.4/go.mod h1:pe4Xr/I+2agvq339vS/OZV+EiHAWtpXQs75rioSW9oA= github.com/474420502/requests v1.6.0 h1:f4h4j40eT0P5whhg9LdkotD8CaKjtuDu/vz9iSUkCgY= github.com/474420502/requests v1.6.0/go.mod h1:SLXrQ5dL9c7dkIeKNUCBAjOIt3J9KFCS2RQjWJecNwo= +github.com/474420502/requests v1.7.0 h1:oaBwVrxZ7yZ+hDOKwHm2NflYib2y1geIUxBxQ2U48mw= +github.com/474420502/requests v1.7.0/go.mod h1:SLXrQ5dL9c7dkIeKNUCBAjOIt3J9KFCS2RQjWJecNwo= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802 h1:1BDTz0u9nC3//pOCMdNH+CiXJVYJh5UQNCOBG7jbELc= diff --git a/store.go b/store.go index da79aa2..82918ad 100644 --- a/store.go +++ b/store.go @@ -382,7 +382,8 @@ func (store *StoreExtractor) UpdateOperator(isource IGet) { // UpdateStreamer Streamer表, 插入数据 func (store *StoreExtractor) UpdateStreamer(streamer IGet) { - _, err := store.db.Exec("UPDATE "+StreamerTable+" SET user_name = ?, live_url = ?, channel = ?, latest_log_uid = ?, tags = ?, ext = ?, operator = ?, update_time = ?, update_interval = ? WHERE uid = ?;", + // log.Printf("UPDATE "+StreamerTable+" SET user_name = %v, live_url = %v, channel = %v, latest_log_uid = %v, tags = %v, ext = %v, operator = %v, update_time = %v, update_interval = %v WHERE uid = %v", streamer.Get("UserName"), streamer.Get("LiveUrl"), streamer.Get("Channel"), streamer.Get("LatestLogUid"), streamer.Get("Tags"), streamer.Get("Ext"), streamer.Get("Operator"), streamer.Get("UpdateTime"), streamer.Get("UpdateInterval"), streamer.Get("Uid")) + _, err := store.db.Exec("UPDATE "+StreamerTable+" SET user_name = ?, live_url = ?, channel = ?, latest_log_uid = ?, tags = ?, ext = ?, operator = ?, update_time = ?, update_interval = ? WHERE uid = ?", streamer.Get("UserName"), streamer.Get("LiveUrl"), streamer.Get("Channel"), streamer.Get("LatestLogUid"), streamer.Get("Tags"), streamer.Get("Ext"), streamer.Get("Operator"), streamer.Get("UpdateTime"), streamer.Get("UpdateInterval"), streamer.Get("Uid")) if err != nil { panic(err) @@ -396,12 +397,14 @@ func (store *StoreExtractor) Update(streamer IGet, fieldvalues ...interface{}) { for i := 0; i < len(fieldvalues); i += 2 { field := fieldvalues[i] values = append(values, fieldvalues[i+1]) - updateSQL += field.(string) + " = ? " + updateSQL += field.(string) + " = ?," } + updateSQL = updateSQL[0 : len(updateSQL)-1] updateSQL += "WHERE uid = ?" values = append(values, streamer.Get("Uid")) _, err := store.db.Exec(updateSQL, values...) if err != nil { + log.Println(updateSQL) panic(err) } } diff --git a/tasks/openrec/openrec_task1/main.go b/tasks/openrec/openrec_task1/main.go index a8f96c1..736ef31 100644 --- a/tasks/openrec/openrec_task1/main.go +++ b/tasks/openrec/openrec_task1/main.go @@ -1,8 +1,5 @@ package main -import "github.com/474420502/hunter" - func main() { - ht := hunter.NewHunter(openrecRanking) - ht.Execute() + Execute() } diff --git a/tasks/openrec/openrec_task1/task_openrec.go b/tasks/openrec/openrec_task1/task_openrec.go index 124be92..7a8774a 100644 --- a/tasks/openrec/openrec_task1/task_openrec.go +++ b/tasks/openrec/openrec_task1/task_openrec.go @@ -4,29 +4,24 @@ import ( "encoding/json" "intimate" "log" - "os" - "os/signal" "strconv" - "sync/atomic" - "syscall" "time" - "github.com/474420502/hunter" + "github.com/474420502/gcurl" "github.com/tidwall/gjson" ) -var openrecRanking *OpenrecRanking - // sstore 源存储实例, 为存储源数据的实现. 表格具体参考sql/intimate_source.sql var sstore *intimate.StoreSource = intimate.NewStoreSource(string(intimate.STOpenrec)) // estore 解析存储连接实例 var estore *intimate.StoreExtractor = intimate.NewStoreExtractor() -func init() { +// Execute 执行方法 +func Execute() { - openrecRanking = &OpenrecRanking{} - openrecRanking.PreCurlUrl = `curl 'https://public.openrec.tv/external/api/v5/channel-ranks?period=monthly&date=&tag=&page=1' \ + ps := intimate.NewPerfectShutdown() + turl := `curl 'https://public.openrec.tv/external/api/v5/channel-ranks?period=monthly&date=&tag=&page=1' \ -H 'authority: public.openrec.tv' \ -H 'accept: application/json, text/javascript, */*; q=0.01' \ -H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36' \ @@ -39,43 +34,25 @@ func init() { -H 'if-none-match: W/"25edb-aUYBdmLqZcr6DW4ZWKX9r2aqolg"' \ --compressed` -} + g := gcurl.ParseRawCURL(turl) + tp := g.Temporary() -// OpenrecRanking 获取排名任务 -type OpenrecRanking struct { - hunter.PreCurlUrl -} + for !ps.IsClose() { -// Execute 执行方法 -func (or *OpenrecRanking) Execute(cxt *hunter.TaskContext) { - - var loop int32 = 1 - - go func() { - signalchan := make(chan os.Signal) - signal.Notify(signalchan, syscall.SIGKILL, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGSTOP) - log.Println("accept stop command:", <-signalchan) - atomic.StoreInt32(&loop, 0) - }() - - for atomic.LoadInt32(&loop) > 0 { - - resp, err := cxt.Hunt() + resp, err := tp.Execute() if err != nil { log.Println(err) time.Sleep(time.Second * 2) continue } - tp := cxt.Temporary() - content := resp.Content() if len(content) <= 200 { //末页时没有内容返回, 末页退出 finishpoint := time.Now() log.Println("任务Ranking UserId结束休眠, 下次启动时间:", finishpoint.Add(time.Minute*120)) for time.Now().Sub(finishpoint) < time.Minute*120 { time.Sleep(time.Second) - if atomic.LoadInt32(&loop) <= 0 { + if ps.IsClose() { return } } @@ -93,13 +70,6 @@ func (or *OpenrecRanking) Execute(cxt *hunter.TaskContext) { for _, User := range result.Array() { userid := User.Get("channel.id").String() - - // data := &intimate.Source{} - // data.Source = sql.NullString{String: userid, Valid: len(userid) > 0} - // data.Url = tp.GetRawURL() - // data.TargetType = string(intimate.TTOpenrecUser) - // sstore.Insert(data) - streamer := &intimate.Streamer{} streamer.UserId = userid streamer.Platform = intimate.Popenrec @@ -123,17 +93,7 @@ func (or *OpenrecRanking) Execute(cxt *hunter.TaskContext) { } // 修改url query 参数的page递增. 遍历所有页面 - querys := tp.GetQuery() - page, err := strconv.Atoi(querys.Get("page")) - if err != nil { - log.Println(err) - return - } - - page++ - querys.Set("page", strconv.Itoa(page)) - tp.SetQuery(querys) - + tp.QueryParam("page").IntAdd(1) time.Sleep(time.Second * 1) } } diff --git a/tasks/openrec/openrec_task1/task_openrec_test.go b/tasks/openrec/openrec_task1/task_openrec_test.go index 7190d66..db355eb 100644 --- a/tasks/openrec/openrec_task1/task_openrec_test.go +++ b/tasks/openrec/openrec_task1/task_openrec_test.go @@ -63,7 +63,6 @@ func TestTimeAdd(t *testing.T) { t.Error(time.Now().Sub(finishpoint) > time.Second*1) } -func TestRankingInsert(t *testing.T) { - ht := hunter.NewHunter(openrecRanking) - ht.Execute() +func TestMain(t *testing.T) { + main() } diff --git a/tasks/openrec/openrec_task2/main.go b/tasks/openrec/openrec_task2/main.go index 4043048..736ef31 100644 --- a/tasks/openrec/openrec_task2/main.go +++ b/tasks/openrec/openrec_task2/main.go @@ -1,8 +1,5 @@ package main -import "github.com/474420502/hunter" - func main() { - ht := hunter.NewHunter(oer) - ht.Execute() + Execute() } diff --git a/tasks/openrec/openrec_task2/task_openrec.go b/tasks/openrec/openrec_task2/task_openrec.go index 9126949..091944f 100644 --- a/tasks/openrec/openrec_task2/task_openrec.go +++ b/tasks/openrec/openrec_task2/task_openrec.go @@ -5,21 +5,13 @@ import ( "encoding/json" "intimate" "log" - "os" - "os/signal" - "strconv" - "sync/atomic" - "syscall" "time" "github.com/474420502/gcurl" + "github.com/474420502/requests" "github.com/tidwall/gjson" - - "github.com/474420502/hunter" ) -var oer *OpenrecExtratorRanking - // sstore 源存储实例, 为存储源数据的实现. 表格具体参考sql/intimate_source.sql var sstore *intimate.StoreSource = intimate.NewStoreSource(string(intimate.STOpenrec)) @@ -27,29 +19,18 @@ var sstore *intimate.StoreSource = intimate.NewStoreSource(string(intimate.STOpe var estore *intimate.StoreExtractor = intimate.NewStoreExtractor() func init() { - oer = &OpenrecExtratorRanking{} -} -// OpenrecExtratorRanking 获取用户信息 -type OpenrecExtratorRanking struct { - // Store *intimate.Store } // Execute 执行方法 -func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) { +func Execute() { - var loop int32 = 1 - - go func() { - signalchan := make(chan os.Signal) - signal.Notify(signalchan, syscall.SIGKILL, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGSTOP) - log.Println("accept stop command:", <-signalchan) - atomic.StoreInt32(&loop, 0) - }() + ps := intimate.NewPerfectShutdown() + ses := requests.NewSession() var lasterr error = nil - for atomic.LoadInt32(&loop) > 0 { + for !ps.IsClose() { streamer, err := estore.Pop(intimate.Popenrec) //队列里弹出一个streamer行. 进行解析 @@ -74,7 +55,8 @@ func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) { // Check Userid userUrl := updateUrl["user"] - tp := cxt.Session().Get(userUrl) // 获取user url页面数据 + log.Println(userUrl) + tp := ses.Get(userUrl) // 获取user url页面数据 resp, err := tp.Execute() streamer.UpdateTime = sql.NullTime{Time: time.Now(), Valid: true} @@ -84,7 +66,7 @@ func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) { continue } - cookies := cxt.Session().GetCookies(tp.GetParsedURL()) + cookies := ses.GetCookies(tp.GetParsedURL()) scurl := updateUrl["supporters"] //获取打赏者的数据 curl := gcurl.ParseRawCURL(scurl) @@ -128,16 +110,17 @@ func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) { } supporters = append(supporters, string(resp.Content())) - page := supportersQuery.Get("page_number") // page_number 加1 - pageint, err := strconv.Atoi(page) - if err != nil { - log.Println(err) - break - } - pageint++ - page = strconv.Itoa(pageint) - supportersQuery.Set("page_number", page) - temporary.SetQuery(supportersQuery) + temporary.QueryParam("page_number").IntAdd(1) + // page := supportersQuery.Get("page_number") // page_number 加1 + // pageint, err := strconv.Atoi(page) + // if err != nil { + // log.Println(err) + // break + // } + // pageint++ + // page = strconv.Itoa(pageint) + // supportersQuery.Set("page_number", page) + // temporary.SetQuery(supportersQuery) } // cookies := cxt.Session().GetCookies(wf.GetParsedURL()) @@ -147,7 +130,7 @@ func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) { ext["html_user"] = string(resp.Content()) liveUrl := updateUrl["live"] - tp = cxt.Session().Get(liveUrl) + tp = ses.Get(liveUrl) resp, err = tp.Execute() if err != nil { log.Println(err) @@ -164,7 +147,10 @@ func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) { continue } - streamer.Operator = int32(intimate.OperatorOK) + // streamer.Platform = intimate.Popenrec + streamer.UpdateInterval = 120 + streamer.UpdateTime = sql.NullTime{Time: time.Now(), Valid: true} + streamer.Operator = 0 source := &intimate.Source{} source.Target = intimate.TOpenrecUser @@ -172,7 +158,7 @@ func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) { source.StreamerId = sql.NullInt64{Int64: streamer.Uid, Valid: true} sstore.Insert(source) - estore.UpdateOperator(streamer) + estore.UpdateStreamer(streamer) } } diff --git a/tasks/openrec/openrec_task2/task_openrec_test.go b/tasks/openrec/openrec_task2/task_openrec_test.go index d29684f..80820ef 100644 --- a/tasks/openrec/openrec_task2/task_openrec_test.go +++ b/tasks/openrec/openrec_task2/task_openrec_test.go @@ -2,11 +2,8 @@ package main import ( "testing" - - "github.com/474420502/hunter" ) -func TestOpenrecUser(t *testing.T) { - ht := hunter.NewHunter(oer) - ht.Execute() +func TestMain(t *testing.T) { + main() } diff --git a/tasks/twitch/twitch_task1/main.go b/tasks/twitch/twitch_task1/main.go index b0019dd..736ef31 100644 --- a/tasks/twitch/twitch_task1/main.go +++ b/tasks/twitch/twitch_task1/main.go @@ -1,6 +1,5 @@ package main func main() { - e := ChannelLink{} - e.Execute() + Execute() } diff --git a/tasks/twitch/twitch_task1/task_twitch.go b/tasks/twitch/twitch_task1/task_twitch.go index d98b28b..03fcaf2 100644 --- a/tasks/twitch/twitch_task1/task_twitch.go +++ b/tasks/twitch/twitch_task1/task_twitch.go @@ -17,12 +17,8 @@ var estore *intimate.StoreExtractor = intimate.NewStoreExtractor() // 获取类型的所有频道链接 -// ChannelLink 频道链接 -type ChannelLink struct { -} - // Execute 执行任务 -func (cl *ChannelLink) Execute() { +func Execute() { var err error wd := intimate.GetChromeDriver(3030) ps := intimate.NewPerfectShutdown() diff --git a/tasks/twitch/twitch_task1/task_twitch_test.go b/tasks/twitch/twitch_task1/task_twitch_test.go index e61dc1e..6ee9af6 100644 --- a/tasks/twitch/twitch_task1/task_twitch_test.go +++ b/tasks/twitch/twitch_task1/task_twitch_test.go @@ -4,9 +4,8 @@ import ( "testing" ) -func TestCase1(t *testing.T) { - e := ChannelLink{} - e.Execute() +func estCase1(t *testing.T) { + Execute() } func TestLiveUrl(t *testing.T) { diff --git a/tasks/twitch/twitch_task2/main.go b/tasks/twitch/twitch_task2/main.go index d81b18b..e86b640 100644 --- a/tasks/twitch/twitch_task2/main.go +++ b/tasks/twitch/twitch_task2/main.go @@ -1,6 +1,6 @@ package main func main() { - ul := UserList{} - ul.Execute() + + Execute() } diff --git a/tasks/twitch/twitch_task2/task_twitch.go b/tasks/twitch/twitch_task2/task_twitch.go index 915a969..922d1b1 100644 --- a/tasks/twitch/twitch_task2/task_twitch.go +++ b/tasks/twitch/twitch_task2/task_twitch.go @@ -19,17 +19,12 @@ var estore *intimate.StoreExtractor = intimate.NewStoreExtractor() // 获取类型的所有频道链接 -// UserList 频道链接 -type UserList struct { -} - // Execute 执行任务 -func (cl *UserList) Execute() { +func Execute() { // DELETE FROM source_twitch WHERE uid NOT IN (SELECT MAX(s.uid) FROM (SELECT uid, source FROM source_twitch ) s GROUP BY s.source) ; //article//a[@data-a-target='preview-card-title-link'] wd := intimate.GetChromeDriver(3030) - defer wd.Close() defer wd.Quit() ps := intimate.NewPerfectShutdown() counter := intimate.NewCounter() @@ -174,7 +169,4 @@ func (cl *UserList) Execute() { } counter.AddWithReset(1) } - - wd.Close() - wd.Quit() } diff --git a/tasks/twitch/twitch_task2/task_twitch_test.go b/tasks/twitch/twitch_task2/task_twitch_test.go index ef64976..a5537f5 100644 --- a/tasks/twitch/twitch_task2/task_twitch_test.go +++ b/tasks/twitch/twitch_task2/task_twitch_test.go @@ -2,6 +2,6 @@ package main import "testing" -func TestMain(t *testing.T) { +func estMain(t *testing.T) { main() } From a50c879d83a4fba2110c0acbe6fbf1f97c3e7859 Mon Sep 17 00:00:00 2001 From: eson Date: Fri, 14 Aug 2020 19:26:03 +0800 Subject: [PATCH 4/8] =?UTF-8?q?todo:=20openrec=20=E9=87=8D=E6=9E=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- extractor/openrec_extractor/openrec_extractor.go | 10 ++++++++++ .../twitcasting_extractor/twitcasting_extractor.go | 4 ++-- go.mod | 2 +- go.sum | 8 ++++++-- tasks/twitcasting/twitcasting_task1/twitcasting.go | 2 +- utils.go | 8 +++++--- 6 files changed, 25 insertions(+), 9 deletions(-) diff --git a/extractor/openrec_extractor/openrec_extractor.go b/extractor/openrec_extractor/openrec_extractor.go index b8a8943..4ac446f 100644 --- a/extractor/openrec_extractor/openrec_extractor.go +++ b/extractor/openrec_extractor/openrec_extractor.go @@ -10,6 +10,7 @@ import ( "strings" "time" + "github.com/474420502/extractor" "github.com/tidwall/gjson" ) @@ -23,6 +24,12 @@ type OpenrecExtractor struct { supporters *intimate.ExtractorSource } +type UserInfo struct { + UserName string `exp:"//p[ contains(@class, 'c-global__user__profile__list__name__text')]"` + Followers int `exp:"//p[@class='c-global__user__count__row__right js-userCountFollowers']" mth:"r:ParseNumber"` + Views int `exp:"//ul[@class='c-contents']//p[@class='c-thumbnailVideo__footer__liveCount']" mth:"r:ExtractNumber"` +} + func (oe *OpenrecExtractor) Execute() { ps := intimate.NewPerfectShutdown() @@ -55,6 +62,9 @@ func (oe *OpenrecExtractor) Execute() { oe.user = intimate.NewExtractorSource(&htmlUser) oe.user.CreateExtractor() + userEtor := extractor.ExtractHtmlString(htmlUser.String()) + log.Println(userEtor.GetObjectByTag(UserInfo{})) + htmlLive := datamap["html_live"] oe.userLive = intimate.NewExtractorSource(&htmlLive) oe.userLive.CreateExtractor() diff --git a/extractor/twitcasting_extractor/twitcasting_extractor.go b/extractor/twitcasting_extractor/twitcasting_extractor.go index 6351c3d..9f0b031 100644 --- a/extractor/twitcasting_extractor/twitcasting_extractor.go +++ b/extractor/twitcasting_extractor/twitcasting_extractor.go @@ -38,7 +38,7 @@ func main() { streamer, err := estore.Pop(intimate.Ptwitcasting) if err != nil { - log.Println(err, streamer.UserId) + log.Println(err, streamer) } streamer.LiveUrl = sql.NullString{String: "https://twitcasting.tv/" + streamer.UserId, Valid: true} @@ -93,7 +93,7 @@ func main() { streamer.Platform = intimate.Ptwitcasting streamer.UpdateTime = sql.NullTime{Time: time.Now(), Valid: true} streamer.UserName = sql.NullString{String: ldata.UserName, Valid: true} - streamer.Operator = 10 + streamer.Operator = 0 // streamer.UpdateInterval = 60 clog := &intimate.CollectLog{} clog.UserId = streamer.UserId diff --git a/go.mod b/go.mod index 36800bf..16c756a 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module intimate go 1.14 require ( - github.com/474420502/extractor v0.7.2 + github.com/474420502/extractor v0.9.4-0.20200814111732-bc270321f8f9 github.com/474420502/focus v0.12.0 github.com/474420502/gcurl v0.2.0 github.com/474420502/hunter v0.3.4 diff --git a/go.sum b/go.sum index 5ed6621..1573290 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,12 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.41.0/go.mod h1:OauMR7DV8fzvZIl2qg6rkaIhD/vmgk4iwEw/h6ercmg= -github.com/474420502/extractor v0.7.2 h1:idZnsekOKRV8fpJwsRcr6Ol7KSphKXe9tc+JJXgGqQ4= -github.com/474420502/extractor v0.7.2/go.mod h1:92J6QZKstpAKGhv+DibemhQbR/d6lJ+ftyR/ZHmeJ0w= +github.com/474420502/extractor v0.9.2 h1:pDDLpxq4bsWjwU3H8epBxDifnmMVBdT0onye2MXCJ8A= +github.com/474420502/extractor v0.9.2/go.mod h1:Ss0KTfwsdB4XBpNda/V50rx21V9bl6/eQmyl50mjAS4= +github.com/474420502/extractor v0.9.3 h1:Cjri64DbgWQQ64EjPiBSQfUH9l0cYlzU8py0PQu8pAs= +github.com/474420502/extractor v0.9.3/go.mod h1:Ss0KTfwsdB4XBpNda/V50rx21V9bl6/eQmyl50mjAS4= +github.com/474420502/extractor v0.9.4-0.20200814111732-bc270321f8f9 h1:cxgsTQwRJSiML4yBL40n/0pD/FbEqkCIXE7qq6hJyLg= +github.com/474420502/extractor v0.9.4-0.20200814111732-bc270321f8f9/go.mod h1:Ss0KTfwsdB4XBpNda/V50rx21V9bl6/eQmyl50mjAS4= github.com/474420502/focus v0.12.0 h1:+icbmj7IEOefvTegHt5EpcHt6WFbe2miIrceUJx2Evo= github.com/474420502/focus v0.12.0/go.mod h1:d0PMjtMxFz1a9HIhwyFPkWa+JF+0LgOrEUfd8iZka6s= github.com/474420502/gcurl v0.1.2 h1:ON9Yz3IgAdtDlFlHfkAJ3aIEBDxH0RiViPE5ST5ohKg= diff --git a/tasks/twitcasting/twitcasting_task1/twitcasting.go b/tasks/twitcasting/twitcasting_task1/twitcasting.go index 89204c2..a11e724 100644 --- a/tasks/twitcasting/twitcasting_task1/twitcasting.go +++ b/tasks/twitcasting/twitcasting_task1/twitcasting.go @@ -89,7 +89,7 @@ func Execute() { continue } - var splist = xps.ForEachTag(SearchProfile{}) + var splist = xps.ForEachObjectByTag(SearchProfile{}) for _, isp := range splist { sp := isp.(*SearchProfile) if sp.LiveUrl == "" { diff --git a/utils.go b/utils.go index cb7d782..2ed1a79 100644 --- a/utils.go +++ b/utils.go @@ -29,14 +29,16 @@ func init() { } // ParseNumber 去逗号解析数字 -func ParseNumber(number string) (int64, error) { - number = strings.ReplaceAll(number, ",", "") - return strconv.ParseInt(number, 10, 64) +func ParseNumber(num string) (int64, error) { + num = strings.Trim(num, " ") + num = strings.ReplaceAll(num, ",", "") + return strconv.ParseInt(num, 10, 64) } // ParseNumberEx 解析带字符的数字 func ParseNumberEx(num string) (float64, error) { num = strings.Trim(num, " ") + num = strings.ReplaceAll(num, ",", "") last := num[len(num)-1] factor := 1.0 switch { From d1298dc3f374be4f2996c742f2abba8e7ad2c035 Mon Sep 17 00:00:00 2001 From: eson Date: Mon, 17 Aug 2020 13:10:29 +0800 Subject: [PATCH 5/8] =?UTF-8?q?finish:=20=E9=87=8D=E6=9E=84openrec?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- extractor/openrec_extractor/main.go | 3 +- .../openrec_extractor/openrec_extractor.go | 238 ++++++------------ extractor/openrec_extractor/openrec_test.go | 3 +- go.mod | 2 +- go.sum | 2 + 5 files changed, 84 insertions(+), 164 deletions(-) diff --git a/extractor/openrec_extractor/main.go b/extractor/openrec_extractor/main.go index 4596421..469a80d 100644 --- a/extractor/openrec_extractor/main.go +++ b/extractor/openrec_extractor/main.go @@ -15,6 +15,5 @@ import ( */ func main() { - oe := &OpenrecExtractor{} - oe.Execute() + Execute() } diff --git a/extractor/openrec_extractor/openrec_extractor.go b/extractor/openrec_extractor/openrec_extractor.go index 4ac446f..4bd51df 100644 --- a/extractor/openrec_extractor/openrec_extractor.go +++ b/extractor/openrec_extractor/openrec_extractor.go @@ -3,11 +3,9 @@ package main import ( "database/sql" "encoding/json" + "errors" "intimate" "log" - "regexp" - "strconv" - "strings" "time" "github.com/474420502/extractor" @@ -17,20 +15,23 @@ import ( var estore = intimate.NewStoreExtractor() var sstore = intimate.NewStoreSource(string(intimate.STOpenrec)) -// OpenrecExtractor 提取方法 -type OpenrecExtractor struct { - user *intimate.ExtractorSource - userLive *intimate.ExtractorSource - supporters *intimate.ExtractorSource -} - +//UserInfo 提取信息的结构体 type UserInfo struct { UserName string `exp:"//p[ contains(@class, 'c-global__user__profile__list__name__text')]"` - Followers int `exp:"//p[@class='c-global__user__count__row__right js-userCountFollowers']" mth:"r:ParseNumber"` - Views int `exp:"//ul[@class='c-contents']//p[@class='c-thumbnailVideo__footer__liveCount']" mth:"r:ExtractNumber"` + Followers int64 `exp:"//p[@class='c-global__user__count__row__right js-userCountFollowers']" mth:"r:ParseNumber"` + Views int64 `exp:"//ul[@class='c-contents']//p[@class='c-thumbnailVideo__footer__liveCount']" mth:"r:ExtractNumber"` } -func (oe *OpenrecExtractor) Execute() { +//UserLive 提取信息的结构体 +type UserLive struct { + Title string `exp:"//h1[contains(@class,'MovieTitle__Title')]"` + LiveStartTime string `exp:"//meta[@itemprop='uploadDate']/@content"` + LiveEndTime string `exp:"//meta[@itemprop='duration']/@content"` + Tags []string `exp:"//a[contains(@class,'TagButton')]"` +} + +// Execute 执行 +func Execute() { ps := intimate.NewPerfectShutdown() @@ -47,6 +48,7 @@ func (oe *OpenrecExtractor) Execute() { time.Sleep(time.Second * 5) continue } + lasterr = nil sdata := source.Ext.([]byte) datamap := gjson.ParseBytes(sdata).Map() @@ -56,31 +58,80 @@ func (oe *OpenrecExtractor) Execute() { streamer := &intimate.Streamer{} streamer.UserId = userId - // streamer.Platform = intimate.Popenrec + // streamer.Platform = intimate.Popenrec 不需要更新字段 htmlUser := datamap["html_user"] - oe.user = intimate.NewExtractorSource(&htmlUser) - oe.user.CreateExtractor() userEtor := extractor.ExtractHtmlString(htmlUser.String()) - log.Println(userEtor.GetObjectByTag(UserInfo{})) + ui, ok1 := userEtor.GetObjectByTag(UserInfo{}).(*UserInfo) htmlLive := datamap["html_live"] - oe.userLive = intimate.NewExtractorSource(&htmlLive) - oe.userLive.CreateExtractor() + + liveEtor := extractor.ExtractHtmlString(htmlLive.String()) + ul, ok2 := liveEtor.GetObjectByTag(UserLive{}).(*UserLive) jsonSupporters := datamap["json_supporters"] - oe.supporters = intimate.NewExtractorSource(&jsonSupporters) clog := &intimate.CollectLog{} - // log.Println(anchorId) + if ok1 { + clog.Followers = sql.NullInt64{Int64: ui.Followers, Valid: true} + clog.Views = sql.NullInt64{Int64: ui.Views, Valid: true} + if ui.Views != 0 { + clog.IsLiveStreaming = true + } + streamer.UserName = sql.NullString{String: ui.UserName, Valid: true} - oe.extractFollowers(clog) - oe.extractUserName(streamer) - oe.extractViewsAndLiveStreaming(clog) - oe.extractGiversAndGratuity(clog) - oe.extractLive(clog) - oe.extractTags(clog) + giverjson := jsonSupporters + var givers []interface{} + var gratuity int64 = 0 + + for _, v := range giverjson.Array() { + giverSource := gjson.Parse(v.String()) + for _, item := range giverSource.Get("data.items").Array() { + givers = append(givers, item.Map()) + gratuity += item.Get("total_yells").Int() + } + } + + giversbytes, err := json.Marshal(givers) + if err != nil { + log.Println(err) + clog.ErrorMsg = sql.NullString{String: err.Error(), Valid: true} + } else { + clog.Giver = giversbytes + } + + clog.Gratuity = sql.NullInt64{Int64: gratuity, Valid: true} + } else { + log.Println("UserInfo may be not exists") + estore.UpdateError(streamer, errors.New("UserInfo may be not exists")) + continue + } + + //log.Println(ul) + if ok2 { + clog.LiveTitle = sql.NullString{String: ul.Title, Valid: true} + + startTime, err := time.ParseInLocation("2006-01-02T15:04:05Z07:00", ul.LiveStartTime, time.Local) + if err != nil { + log.Println(err) + } else { + clog.LiveStartTime = sql.NullTime{Time: startTime.Local(), Valid: true} + duration, err := intimate.ParseDuration(ul.LiveEndTime) + if err != nil { + log.Println(err) + } else { + endTime := startTime.Add(duration) + clog.LiveStartTime = sql.NullTime{Time: endTime.Local(), Valid: true} + } + } + + if tags, err := json.Marshal(ul.Tags); err == nil { + clog.Tags = tags + } else { + log.Println("json error", ul.Tags, clog.Tags) + } + } streamer.Uid = source.StreamerId.Int64 streamer.UpdateTime = source.UpdateTime @@ -89,6 +140,7 @@ func (oe *OpenrecExtractor) Execute() { clog.Platform = intimate.Popenrec clog.UserId = userId clog.UpdateTime = source.UpdateTime + clog.StreamerUid = streamer.Uid logUid := estore.InsertClog(clog) @@ -113,135 +165,3 @@ func (oe *OpenrecExtractor) Execute() { } } - -func (oe *OpenrecExtractor) extractFollowers(clog intimate.ISet) { - extractor := oe.user.GetExtractor() - xp, err := extractor.XPathResult("//p[@class='c-global__user__count__row__right js-userCountFollowers']/text()") - if err != nil { - log.Println(err) - } - if !xp.NodeIter().Next() { - log.Println("不存在粉丝数") - } - - followers := strings.ReplaceAll(xp.String(), ",", "") - followersInt, err := strconv.ParseInt(followers, 10, 64) - if err != nil { - log.Println(err) - } - - clog.Set("Followers", sql.NullInt64{Int64: followersInt, Valid: true}) -} - -func (oe *OpenrecExtractor) extractUserName(streamer intimate.ISet) { - extractor := oe.user.GetExtractor() - xp, err := extractor.XPathResult("//p[ contains(@class, 'c-global__user__profile__list__name__text')]/text()") - if err != nil { - log.Println(err) - } else { - if xp.NodeIter().Next() { - userName := xp.String() - streamer.Set("UserName", sql.NullString{String: userName, Valid: true}) - } - } -} - -func (oe *OpenrecExtractor) extractViewsAndLiveStreaming(clog intimate.ISet) { - extractor := oe.user.GetExtractor() - // c-contents - xp, err := extractor.XPathResult("//ul[@class='c-contents']//p[@class='c-thumbnailVideo__footer__liveCount']/text()") - if err != nil { - log.Println(err) - } - - if xp.NodeIter().Next() { - views := regexp.MustCompile(`[0-9,]+`).FindString(xp.String()) - views = strings.ReplaceAll(views, ",", "") - viewsint, err := strconv.Atoi(views) - if err != nil { - log.Println(err) - } - - clog.Set("Views", sql.NullInt64{Int64: int64(viewsint), Valid: true}) - clog.Set("IsLiveStreaming", true) - } -} - -func (oe *OpenrecExtractor) extractGiversAndGratuity(clog intimate.ISet) { - // extractor := oe.user.GetExtractor() - giverjson := oe.supporters.GetSource() - var givers []interface{} - var gratuity int64 = 0 - - for _, v := range giverjson.Array() { - giverSource := gjson.Parse(v.String()) - for _, item := range giverSource.Get("data.items").Array() { - givers = append(givers, item.Map()) - gratuity += item.Get("total_yells").Int() - } - } - - giversbytes, err := json.Marshal(givers) - if err != nil { - log.Println(err) - clog.Set("ErrorMsg", sql.NullString{String: err.Error(), Valid: true}) - } else { - clog.Set("Giver", giversbytes) - } - - clog.Set("Gratuity", sql.NullInt64{Int64: gratuity, Valid: true}) -} - -func (oe *OpenrecExtractor) extractLive(clog intimate.ISet) { - extractor := oe.userLive.GetExtractor() - mathes := regexp.MustCompile("MovieTitle__Title[^>]+>(.{1,50})").FindStringSubmatch(oe.userLive.GetSource().Str) - if len(mathes) == 2 { - - clog.Set("LiveTitle", sql.NullString{String: mathes[1], Valid: true}) - - content, err := extractor.XPathResult("//meta[@itemprop='uploadDate']/@content") - if err != nil { - log.Println(err) - } - - iter := content.NodeIter() - if iter.Next() { - tm, err := time.ParseInLocation("2006-01-02T15:04:05Z07:00", iter.Node().NodeValue(), time.Local) - if err != nil { - log.Println(err) - } - // log.Println(iter.Node().NodeValue(), tm.Local()) - clog.Set("LiveStartTime", sql.NullTime{Time: tm.Local(), Valid: true}) - - duration, err := extractor.XPathResult("//meta[@itemprop='duration']/@content") - if err != nil { - log.Println(err) - } - - diter := duration.NodeIter() - if diter.Next() { - - dt, err := intimate.ParseDuration(diter.Node().NodeValue()) - if err != nil { - log.Println(err) - } - endtm := tm.Add(dt) - clog.Set("LiveEndTime", sql.NullTime{Time: endtm.Local(), Valid: true}) - } - } - } -} - -func (oe *OpenrecExtractor) extractTags(clog intimate.ISet) { - var tags []string - matheslist := regexp.MustCompile(`<[^>]+TagButton[^>]+>([^<]{1,100})<`).FindAllStringSubmatch(oe.userLive.GetSource().Str, -1) - for _, m := range matheslist { - tags = append(tags, m[1]) - } - tagsBytes, err := json.Marshal(tags) - if err != nil { - log.Println(err) - } - - clog.Set("Tags", tagsBytes) -} diff --git a/extractor/openrec_extractor/openrec_test.go b/extractor/openrec_extractor/openrec_test.go index 2028e34..e9bc878 100644 --- a/extractor/openrec_extractor/openrec_test.go +++ b/extractor/openrec_extractor/openrec_test.go @@ -114,6 +114,5 @@ func TestUserName(t *testing.T) { } func TestExtractor(t *testing.T) { - oe := &OpenrecExtractor{} - oe.Execute() + Execute() } diff --git a/go.mod b/go.mod index 16c756a..e58f8be 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module intimate go 1.14 require ( - github.com/474420502/extractor v0.9.4-0.20200814111732-bc270321f8f9 + github.com/474420502/extractor v0.9.4-0.20200817020657-7d26da5d1e89 github.com/474420502/focus v0.12.0 github.com/474420502/gcurl v0.2.0 github.com/474420502/hunter v0.3.4 diff --git a/go.sum b/go.sum index 1573290..e71679a 100644 --- a/go.sum +++ b/go.sum @@ -8,6 +8,8 @@ github.com/474420502/extractor v0.9.3 h1:Cjri64DbgWQQ64EjPiBSQfUH9l0cYlzU8py0PQu github.com/474420502/extractor v0.9.3/go.mod h1:Ss0KTfwsdB4XBpNda/V50rx21V9bl6/eQmyl50mjAS4= github.com/474420502/extractor v0.9.4-0.20200814111732-bc270321f8f9 h1:cxgsTQwRJSiML4yBL40n/0pD/FbEqkCIXE7qq6hJyLg= github.com/474420502/extractor v0.9.4-0.20200814111732-bc270321f8f9/go.mod h1:Ss0KTfwsdB4XBpNda/V50rx21V9bl6/eQmyl50mjAS4= +github.com/474420502/extractor v0.9.4-0.20200817020657-7d26da5d1e89 h1:6g4sPgooFdsVAdxNMhP6sqKQ0Z5EPBb4tGj9/absPoY= +github.com/474420502/extractor v0.9.4-0.20200817020657-7d26da5d1e89/go.mod h1:Ss0KTfwsdB4XBpNda/V50rx21V9bl6/eQmyl50mjAS4= github.com/474420502/focus v0.12.0 h1:+icbmj7IEOefvTegHt5EpcHt6WFbe2miIrceUJx2Evo= github.com/474420502/focus v0.12.0/go.mod h1:d0PMjtMxFz1a9HIhwyFPkWa+JF+0LgOrEUfd8iZka6s= github.com/474420502/gcurl v0.1.2 h1:ON9Yz3IgAdtDlFlHfkAJ3aIEBDxH0RiViPE5ST5ohKg= From 9d7c2e1e549c7f9b0c57e7208bf5e3f8abf9b092 Mon Sep 17 00:00:00 2001 From: eson Date: Mon, 17 Aug 2020 14:38:00 +0800 Subject: [PATCH 6/8] =?UTF-8?q?1.=E4=BF=AE=E6=94=B9build.sh=202.=E5=88=A0?= =?UTF-8?q?=E9=99=A4hunter=E5=8C=85=E5=BC=95=E7=94=A8=E5=92=8C=E4=BD=BF?= =?UTF-8?q?=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 1 + build.sh | 8 ++- extractor/openrec_extractor/openrec_test.go | 23 -------- extractor_field.go | 36 ------------- go.mod | 5 +- go.sum | 12 +---- store_test.go | 3 +- .../openrec_task1/task_openrec_test.go | 52 ------------------- 8 files changed, 12 insertions(+), 128 deletions(-) diff --git a/.gitignore b/.gitignore index 657f5f1..23a8cf4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +bin *.log log screenlog.* diff --git a/build.sh b/build.sh index 6bdb63f..f92ccd4 100644 --- a/build.sh +++ b/build.sh @@ -7,13 +7,17 @@ src=`pwd` for path in `ls -d $source_tasks` do echo $path - cd $path && go build + projectname=${path##*/} + projectworkspace=$src/bin/$projectname + cd $path && mkdir $projectworkspace -p && go build -o $projectworkspace/$projectname cd $src done for path in `ls -d $extractor_tasks` do echo $path - cd $path && go build + projectname=${path##*/} + projectworkspace=$src/bin/$projectname + cd $path && mkdir $projectworkspace -p && go build -o $projectworkspace/$projectname cd $src done diff --git a/extractor/openrec_extractor/openrec_test.go b/extractor/openrec_extractor/openrec_test.go index e9bc878..2620803 100644 --- a/extractor/openrec_extractor/openrec_test.go +++ b/extractor/openrec_extractor/openrec_test.go @@ -7,7 +7,6 @@ import ( "testing" "time" - "github.com/474420502/hunter" "github.com/lestrrat-go/libxml2" ) @@ -91,28 +90,6 @@ func TestCase(t *testing.T) { t.Error(xr) } -func TestUserName(t *testing.T) { - f, err := os.Open("test.html") - if err != nil { - panic(err) - } - data, err := ioutil.ReadAll(f) - if err != nil { - panic(err) - } - - extractor := hunter.NewExtractor(data) - xp, err := extractor.XPathResult("//p[ contains(@class, 'c-global__user__profile__list__name__text')]/text()") - if err != nil { - t.Error(err) - } else { - if xp.NodeIter().Next() { - userName := xp.String() - t.Error(userName) - } - } -} - func TestExtractor(t *testing.T) { Execute() } diff --git a/extractor_field.go b/extractor_field.go index 2977941..1adda5e 100644 --- a/extractor_field.go +++ b/extractor_field.go @@ -4,9 +4,6 @@ import ( "database/sql" "reflect" "time" - - "github.com/474420502/hunter" - "github.com/tidwall/gjson" ) type GetSet struct { @@ -104,36 +101,3 @@ func (cl *CollectLog) Get(field string) interface{} { func (cl *CollectLog) Set(field string, value interface{}) { reflect.ValueOf(cl).Elem().FieldByName(field).Set(reflect.ValueOf(value)) } - -type ExtractorSource struct { - source *gjson.Result - extractor *hunter.Extractor -} - -func NewExtractorSource(gr *gjson.Result) *ExtractorSource { - es := &ExtractorSource{} - es.SetSource(gr) - return es -} - -func (es *ExtractorSource) SetSource(gr *gjson.Result) { - es.source = gr - es.extractor = nil -} - -func (es *ExtractorSource) Clear() { - es.source = nil - es.extractor = nil -} - -func (es *ExtractorSource) CreateExtractor() { - es.extractor = hunter.NewExtractor([]byte(es.source.String())) -} - -func (es *ExtractorSource) GetSource() *gjson.Result { - return es.source -} - -func (es *ExtractorSource) GetExtractor() *hunter.Extractor { - return es.extractor -} diff --git a/go.mod b/go.mod index e58f8be..312edd1 100644 --- a/go.mod +++ b/go.mod @@ -1,12 +1,11 @@ module intimate -go 1.14 +go 1.15 require ( - github.com/474420502/extractor v0.9.4-0.20200817020657-7d26da5d1e89 + github.com/474420502/extractor v0.9.5 github.com/474420502/focus v0.12.0 github.com/474420502/gcurl v0.2.0 - github.com/474420502/hunter v0.3.4 github.com/474420502/requests v1.7.0 github.com/go-sql-driver/mysql v1.5.0 github.com/lestrrat-go/libxml2 v0.0.0-20200215080510-6483566f52cb diff --git a/go.sum b/go.sum index e71679a..65f21fa 100644 --- a/go.sum +++ b/go.sum @@ -2,14 +2,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= cloud.google.com/go v0.41.0/go.mod h1:OauMR7DV8fzvZIl2qg6rkaIhD/vmgk4iwEw/h6ercmg= -github.com/474420502/extractor v0.9.2 h1:pDDLpxq4bsWjwU3H8epBxDifnmMVBdT0onye2MXCJ8A= -github.com/474420502/extractor v0.9.2/go.mod h1:Ss0KTfwsdB4XBpNda/V50rx21V9bl6/eQmyl50mjAS4= -github.com/474420502/extractor v0.9.3 h1:Cjri64DbgWQQ64EjPiBSQfUH9l0cYlzU8py0PQu8pAs= -github.com/474420502/extractor v0.9.3/go.mod h1:Ss0KTfwsdB4XBpNda/V50rx21V9bl6/eQmyl50mjAS4= -github.com/474420502/extractor v0.9.4-0.20200814111732-bc270321f8f9 h1:cxgsTQwRJSiML4yBL40n/0pD/FbEqkCIXE7qq6hJyLg= -github.com/474420502/extractor v0.9.4-0.20200814111732-bc270321f8f9/go.mod h1:Ss0KTfwsdB4XBpNda/V50rx21V9bl6/eQmyl50mjAS4= -github.com/474420502/extractor v0.9.4-0.20200817020657-7d26da5d1e89 h1:6g4sPgooFdsVAdxNMhP6sqKQ0Z5EPBb4tGj9/absPoY= -github.com/474420502/extractor v0.9.4-0.20200817020657-7d26da5d1e89/go.mod h1:Ss0KTfwsdB4XBpNda/V50rx21V9bl6/eQmyl50mjAS4= +github.com/474420502/extractor v0.9.5 h1:nM3/1tNL8BHS3PN9eXvm8Ve3hqTReKNB+ba8VpdL8bw= +github.com/474420502/extractor v0.9.5/go.mod h1:vJnXWmvO5bJDW4Yag0GoE2GxtHRg03TAxp2oXN1DcSY= github.com/474420502/focus v0.12.0 h1:+icbmj7IEOefvTegHt5EpcHt6WFbe2miIrceUJx2Evo= github.com/474420502/focus v0.12.0/go.mod h1:d0PMjtMxFz1a9HIhwyFPkWa+JF+0LgOrEUfd8iZka6s= github.com/474420502/gcurl v0.1.2 h1:ON9Yz3IgAdtDlFlHfkAJ3aIEBDxH0RiViPE5ST5ohKg= @@ -18,8 +12,6 @@ github.com/474420502/gcurl v0.2.0 h1:m6+vw4NX4f5Tfp7c3nuaIgHUE/7zTX6K3xK+pTCBoCo github.com/474420502/gcurl v0.2.0/go.mod h1:kJZDbgXn5wbAaR+hhBi4Sbw44P4igJ7qYXC6mejLuhQ= github.com/474420502/htmlquery v1.2.4-0.20200812072201-e871dd09247a h1:E1T6CYQKsUn7fMvNbeKfISjBLfOJjZX4KpWwStT20Kc= github.com/474420502/htmlquery v1.2.4-0.20200812072201-e871dd09247a/go.mod h1:AoSN890esHwNKecV0tCs+W0ele1xgFL1Jqk6UcrdxgU= -github.com/474420502/hunter v0.3.4 h1:fyLAgI84jWe3IcqsISC53j1w3CXI1FERxX//Potns0M= -github.com/474420502/hunter v0.3.4/go.mod h1:pe4Xr/I+2agvq339vS/OZV+EiHAWtpXQs75rioSW9oA= github.com/474420502/requests v1.6.0 h1:f4h4j40eT0P5whhg9LdkotD8CaKjtuDu/vz9iSUkCgY= github.com/474420502/requests v1.6.0/go.mod h1:SLXrQ5dL9c7dkIeKNUCBAjOIt3J9KFCS2RQjWJecNwo= github.com/474420502/requests v1.7.0 h1:oaBwVrxZ7yZ+hDOKwHm2NflYib2y1geIUxBxQ2U48mw= diff --git a/store_test.go b/store_test.go index 7d169ff..d9c6858 100644 --- a/store_test.go +++ b/store_test.go @@ -5,8 +5,7 @@ import ( ) func TestStoreInsert(t *testing.T) { - // ht := hunter.NewHunter(openrecRanking) - // ht.Execute() + } func TestStoreInsertCase1(t *testing.T) { diff --git a/tasks/openrec/openrec_task1/task_openrec_test.go b/tasks/openrec/openrec_task1/task_openrec_test.go index db355eb..9e649d2 100644 --- a/tasks/openrec/openrec_task1/task_openrec_test.go +++ b/tasks/openrec/openrec_task1/task_openrec_test.go @@ -3,60 +3,8 @@ package main import ( "testing" "time" - - "github.com/tidwall/gjson" - - "github.com/474420502/hunter" ) -// OpenrecRanking 获取排名任务 -type OpenrecRankingTest struct { - hunter.PreCurlUrl -} - -// Execute 执行方法 -func (or *OpenrecRankingTest) Execute(cxt *hunter.TaskContext) { - - resp, err := cxt.Hunt() - if err != nil { - panic(err) - } - t := cxt.GetShare("test").(*testing.T) - if !gjson.ValidBytes(resp.Content()) { - t.Error("source is not json format.") - } - result := gjson.GetBytes(resp.Content(), "0.rank") - if result.Int() != 1 { - t.Error("rank is error. result raw is ", result.Raw) - } - - if cxt.Temporary().GetQuery().Get("page") != "1" { - t.Error("Temporary page error") - } - // t.Error(string(resp.Content())) -} - -func TestRanking(t *testing.T) { - curlBash := `curl 'https://public.openrec.tv/external/api/v5/channel-ranks?period=monthly&date=&tag=&page=1' \ - -H 'authority: public.openrec.tv' \ - -H 'accept: application/json, text/javascript, */*; q=0.01' \ - -H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36' \ - -H 'origin: https://www.openrec.tv' \ - -H 'sec-fetch-site: same-site' \ - -H 'sec-fetch-mode: cors' \ - -H 'sec-fetch-dest: empty' \ - -H 'referer: https://www.openrec.tv/ranking' \ - -H 'accept-language: zh-CN,zh;q=0.9' \ - -H 'if-none-match: W/"25edb-aUYBdmLqZcr6DW4ZWKX9r2aqolg"' \ - --compressed` - - ort := &OpenrecRankingTest{} - ort.PreCurlUrl = hunter.PreCurlUrl(curlBash) - ht := hunter.NewHunter(ort) - ht.SetShare("test", t) - ht.Execute() -} - func TestTimeAdd(t *testing.T) { finishpoint := time.Now() time.Sleep(time.Second * 2) From 61634751bf8c783e894e86309ffad7080245d41c Mon Sep 17 00:00:00 2001 From: eson Date: Mon, 17 Aug 2020 14:50:19 +0800 Subject: [PATCH 7/8] add supervisor conf --- supervisor_conf/openrec_extractor.conf | 10 ++++++++++ supervisor_conf/openrec_task1.conf | 10 ++++++++++ supervisor_conf/openrec_task2.conf | 12 ++++++++++++ supervisor_conf/test.conf | 12 ++++++++++++ supervisor_conf/twitcasting_extractor.conf | 14 ++++++++++++++ supervisor_conf/twitcasting_task1.conf | 13 +++++++++++++ supervisor_conf/twitch_extractor.conf | 13 +++++++++++++ supervisor_conf/twitch_extractor_p1.conf | 13 +++++++++++++ supervisor_conf/twitch_extractor_p2.conf | 13 +++++++++++++ supervisor_conf/twitch_task2.conf | 13 +++++++++++++ supervisor_conf/xvfb.conf | 13 +++++++++++++ 11 files changed, 136 insertions(+) create mode 100644 supervisor_conf/openrec_extractor.conf create mode 100644 supervisor_conf/openrec_task1.conf create mode 100644 supervisor_conf/openrec_task2.conf create mode 100644 supervisor_conf/test.conf create mode 100644 supervisor_conf/twitcasting_extractor.conf create mode 100644 supervisor_conf/twitcasting_task1.conf create mode 100644 supervisor_conf/twitch_extractor.conf create mode 100644 supervisor_conf/twitch_extractor_p1.conf create mode 100644 supervisor_conf/twitch_extractor_p2.conf create mode 100644 supervisor_conf/twitch_task2.conf create mode 100644 supervisor_conf/xvfb.conf diff --git a/supervisor_conf/openrec_extractor.conf b/supervisor_conf/openrec_extractor.conf new file mode 100644 index 0000000..0b7f9de --- /dev/null +++ b/supervisor_conf/openrec_extractor.conf @@ -0,0 +1,10 @@ +[supervisord] +nodaemon=true + +[program:openrec_extractor] +directory = /home/eson/test/intimate/bin/openrec_extractor/ +command= /home/eson/test/intimate/bin/openrec_extractor/openrec_extractor +autorestart=true +stderr_logfile=/home/eson/test/intimate/bin/openrec_extractor/log +stderr_logfile_maxbytes=0 +stopsignal=QUIT diff --git a/supervisor_conf/openrec_task1.conf b/supervisor_conf/openrec_task1.conf new file mode 100644 index 0000000..b011692 --- /dev/null +++ b/supervisor_conf/openrec_task1.conf @@ -0,0 +1,10 @@ +[supervisord] +nodaemon=true + +[program:openrec_ranking] +directory = /home/eson/test/intimate/bin/openrec_task1 +command= /home/eson/test/intimate/bin/openrec_task1/openrec_task1 +autorestart=true +stderr_logfile=/home/eson/test/intimate/bin/openrec_task1/log +stderr_logfile_maxbytes=0 +stopsignal=QUIT diff --git a/supervisor_conf/openrec_task2.conf b/supervisor_conf/openrec_task2.conf new file mode 100644 index 0000000..99c65c3 --- /dev/null +++ b/supervisor_conf/openrec_task2.conf @@ -0,0 +1,12 @@ +[supervisord] +nodaemon=true + +[program:openrec_source] +directory = /home/eson/test/intimate/bin/openrec_task2/ +command= /home/eson/test/intimate/bin/openrec_task2/openrec_task2 +process_name=%(program_name)s_%(process_num)02d ;多进程名称 +numprocs=4 ;启动多个进程 +autorestart=true +stderr_logfile=/home/eson/test/intimate/bin/openrec_task2/log +stderr_logfile_maxbytes=0 +stopsignal=QUIT diff --git a/supervisor_conf/test.conf b/supervisor_conf/test.conf new file mode 100644 index 0000000..fb815c7 --- /dev/null +++ b/supervisor_conf/test.conf @@ -0,0 +1,12 @@ +# [supervisord] +# nodaemon=true + +# [program:openrec_extractor] +# directory = /home/eson/test/intimate/extractor/openrec_extractor +# command= /home/eson/test/intimate/extractor/openrec_extractor/openrec_extractor +# autorestart=true +# # stdout_logfile=/home/eson/test/intimate/extractor/openrec/stdout +# # stdout_logfile_maxbytes=0 +# stderr_logfile=/home/eson/test/intimate/extractor/openrec/log +# stderr_logfile_maxbytes=0 +# stopsignal=QUIT diff --git a/supervisor_conf/twitcasting_extractor.conf b/supervisor_conf/twitcasting_extractor.conf new file mode 100644 index 0000000..218b2b8 --- /dev/null +++ b/supervisor_conf/twitcasting_extractor.conf @@ -0,0 +1,14 @@ +[supervisord] +nodaemon=false + +[program:twitcasting_extractor] +environment=DISPLAY=":99" +directory = /home/eson/test/intimate/bin/twitcasting_extractor/ +command= /home/eson/test/intimate/bin/twitcasting_extractor/twitcasting_extractor +process_name=%(program_name)s_%(process_num)02d ;多进程名称 +numprocs=3 ;启动多个进程 +autorestart=true +stderr_logfile=/home/eson/test/intimate/bin/twitcasting_extractor/log +# stderr_logfile=%(supervisorctl.var.directory)s/log +stderr_logfile_maxbytes=0 +stopsignal=QUIT diff --git a/supervisor_conf/twitcasting_task1.conf b/supervisor_conf/twitcasting_task1.conf new file mode 100644 index 0000000..23dda7b --- /dev/null +++ b/supervisor_conf/twitcasting_task1.conf @@ -0,0 +1,13 @@ +[supervisord] +nodaemon=false + +[program:twitcasting_task1] +environment=DISPLAY=":99" +directory = /home/eson/test/intimate/bin/twitcasting_task1/ +command= /home/eson/test/intimate/bin/twitcasting_task1/twitcasting_task1 +# process_name=%(program_name)s_%(process_num)02d ;多进程名称 +# numprocs=1 ;启动多个进程 +autorestart=true +stderr_logfile=/home/eson/test/intimate/bin/twitcasting_task1/log +stderr_logfile_maxbytes=0 +stopsignal=QUIT diff --git a/supervisor_conf/twitch_extractor.conf b/supervisor_conf/twitch_extractor.conf new file mode 100644 index 0000000..a8676a5 --- /dev/null +++ b/supervisor_conf/twitch_extractor.conf @@ -0,0 +1,13 @@ +[supervisord] +nodaemon=false + +[program:twitch_extractor] +environment=DISPLAY=":99" +directory = /home/eson/test/intimate/bin/twitch_extractor +command= /home/eson/test/intimate/bin/twitch_extractor/twitch_extractor +process_name=%(program_name)s_%(process_num)02d ;多进程名称 +numprocs=5 ;启动多个进程 +autorestart=true +stderr_logfile=/home/eson/test/intimate/bin/twitch_extractor/log +stderr_logfile_maxbytes=0 +stopsignal=QUIT diff --git a/supervisor_conf/twitch_extractor_p1.conf b/supervisor_conf/twitch_extractor_p1.conf new file mode 100644 index 0000000..36ac3fc --- /dev/null +++ b/supervisor_conf/twitch_extractor_p1.conf @@ -0,0 +1,13 @@ +[supervisord] +nodaemon=true + +[program:twitch_extractor_p1] +environment=DISPLAY=":99",pac_proxy=http://localhost:1090/pac +directory = /home/eson/test/intimate/bin/twitch_extractor +command= /home/eson/test/intimate/bin/twitch_extractor/twitch_extractor +process_name=%(program_name)s_%(process_num)02d ;多进程名称 +numprocs=2 ;启动多个进程 +autorestart=true +stderr_logfile=/home/eson/test/intimate/bin/twitch_extractor/log +stderr_logfile_maxbytes=0 +stopsignal=QUIT diff --git a/supervisor_conf/twitch_extractor_p2.conf b/supervisor_conf/twitch_extractor_p2.conf new file mode 100644 index 0000000..095de77 --- /dev/null +++ b/supervisor_conf/twitch_extractor_p2.conf @@ -0,0 +1,13 @@ +[supervisord] +nodaemon=false + +[program:twitch_extractor_p2] +environment=DISPLAY=":99",pac_proxy=http://localhost:1090/pac1 +directory = /home/eson/test/intimate/bin/twitch_extractor +command= /home/eson/test/intimate/bin/twitch_extractor/twitch_extractor +process_name=%(program_name)s_%(process_num)02d ;多进程名称 +numprocs=2 ;启动多个进程 +autorestart=true +stderr_logfile=/home/eson/test/intimate/bin/twitch_extractor/log +stderr_logfile_maxbytes=0 +stopsignal=QUIT diff --git a/supervisor_conf/twitch_task2.conf b/supervisor_conf/twitch_task2.conf new file mode 100644 index 0000000..7768d0e --- /dev/null +++ b/supervisor_conf/twitch_task2.conf @@ -0,0 +1,13 @@ +[supervisord] +nodaemon=false + +[program:twitch_task2] +environment=DISPLAY=":99" +directory = /home/eson/test/intimate/bin/twitch_task2 +command= /home/eson/test/intimate/bin/twitch_task2/twitch_task2 +process_name=%(program_name)s_%(process_num)02d ;多进程名称 +numprocs=6 ;启动多个进程 +autorestart=true +stderr_logfile=/home/eson/test/intimate/bin/twitch_task2/log +stderr_logfile_maxbytes=0 +stopsignal=QUIT diff --git a/supervisor_conf/xvfb.conf b/supervisor_conf/xvfb.conf new file mode 100644 index 0000000..1ee51d6 --- /dev/null +++ b/supervisor_conf/xvfb.conf @@ -0,0 +1,13 @@ + +[supervisord] +nodaemon=true + +[program:xvfb-99] +# directory = /home/eson/test/intimate/extractor/openrec_extractor +command=/usr/bin/Xvfb :99 -screen 0 1280x720x24 -ac -nolisten tcp -dpi 96 +extension RANDR -nolisten tcp +autorestart=true +# # stdout_logfile=/home/eson/test/intimate/extractor/openrec/stdout +# # stdout_logfile_maxbytes=0 +# stderr_logfile=/home/eson/test/intimate/extractor/openrec/log +# stderr_logfile_maxbytes=0 +# stopsignal=QUIT From 5a8eda6f1b1e7a6014d73522d92f10b5968f47b5 Mon Sep 17 00:00:00 2001 From: eson Date: Mon, 17 Aug 2020 14:52:21 +0800 Subject: [PATCH 8/8] create ln to supervisor conf.d path --- supervisor_conf/createln.sh | 1 + 1 file changed, 1 insertion(+) create mode 100644 supervisor_conf/createln.sh diff --git a/supervisor_conf/createln.sh b/supervisor_conf/createln.sh new file mode 100644 index 0000000..1c092ed --- /dev/null +++ b/supervisor_conf/createln.sh @@ -0,0 +1 @@ +ln -sf `pwd`/*.conf /etc/supervisor/conf.d/