+
+
+
+
+
+
+
+
+
+
+
+
+
+ + Tag Search : + vocaloid
+ + + Live(0) / + User(28) + + + + + + +
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+ 
+
+
+
+
+ 
+ 
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+ 
+
+
+
+ んぁゆ (NaYu)
+
+
+ @NaYu_NotYou
🇮🇩🛫🇦🇺 || Main Account || Korean Related: @NaYu_ha || Pull me into more hells- || icon: @AyaminTwT
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+ f:Egao No ShouJou
+
+
+ @f:100000242133373
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+ f:Yuu'll Be Back
+
+
+ @f:100004051574775
+ 
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+ f:Thamanan Wittayachamnankul
+
+
+ @f:100000714321056
อ่าา ชอบเกรียนที่สุดเบยย อ๊ายยย ขอให้ได้เกรียน เกรียนในสิ่งที่ชอบ และไม่เดือดร้อนคนอื่น โอเคโน๊ะ อยากจะเกรียนเพลงใส่กันเชิญจ้า กร๊ากก ผมเกรียนได้ทุกคนอยู่แระ ..
+
+ 
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+ 
+
+
+
+ totally katië 123
+
+
+ @katie_diva_xoxo
❤️❤️❤️zoella
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+ 
+
+
+
+ Fujisaki Hitomi
+
+
+ @c:vlemvpe
Fujisaki Hitomi
+https://www.facebook.com/ciazfah2
+คนไทยค่า :D
+
+ 
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+ 
+
+
+
+ カワズ先輩≠カエル先輩
+
+
+ @kawazu1816
愛したって、愛されたっていいじゃないか
+
+
+
+
+
+ 
+ 
+
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+ f:Xio Steph
+
+
+ @f:100007528892324
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+
+
+ f:Kama Jewell Greco
+
+
+ @f:489672901190991
+
+
+
+
+
+
+
+
+
+ 
+
+
+ 
+
+
+
+ 英 姫柳/かんじむずかしい
+
+
+ @Kiryu_Hanabusa
YouTuberになるために準備中です。 基本受けの体制なので話し掛けてもらうまでROMってることが多いです。気軽にお声掛けください！ codevein/競馬/麻雀/雀魂/FPS #姫柳の..
+
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+ 
+
+
+
+ ムジャオ＠3/9アイコン変わりました
+
+
+ @mujamiku
ただのザコなボカロファンです。
+
+
+
+
+ 
+
+
+
+
+
+
+
+
+
+
+ 
+
+
+ 
+
+
+
+ miming(みみん)
+
+
+ @c:mimingdayo
韓国人です。日本語勉強はじめたのは10年くらい前からで
+留学はしたことありません。
+まぁ気ままにやります。
+ヾ(⌒(_*'ω'*)_
+
+ 
+
+
+
+
+
+
+
+
+
diff --git a/tasks/twitcasting/twitcasting_task1/main.go b/tasks/twitcasting/twitcasting_task1/main.go
new file mode 100644
index 0000000..736ef31
--- /dev/null
+++ b/tasks/twitcasting/twitcasting_task1/main.go
@@ -0,0 +1,5 @@
+package main
+
+func main() {
+ Execute()
+}
diff --git a/tasks/twitcasting/twitcasting_task1/main_test.go b/tasks/twitcasting/twitcasting_task1/main_test.go
new file mode 100644
index 0000000..18bfb6a
--- /dev/null
+++ b/tasks/twitcasting/twitcasting_task1/main_test.go
@@ -0,0 +1,25 @@
+package main
+
+import (
+ "net/url"
+
+ "testing"
+)
+
+func Test(t *testing.T) {
+ rawurl := "https://twitcasting.tv/你好"
+ u, _ := url.Parse(rawurl)
+ t.Error(u.EscapedPath())
+ t.Error(u.String())
+}
+
+func TestUpdateTime(t *testing.T) {
+ // streamer := &intimate.Streamer{}
+ // streamer.Uid = 420153
+ // streamer.UpdateTime = sql.NullTime{Time: time.Now(), Valid: true}
+ // estore.Update(streamer, "update_time", streamer.UpdateTime)
+}
+
+func TestMain(t *testing.T) {
+
+}
diff --git a/tasks/twitcasting/twitcasting_task1/twitcasting.go b/tasks/twitcasting/twitcasting_task1/twitcasting.go
new file mode 100644
index 0000000..58e2cba
--- /dev/null
+++ b/tasks/twitcasting/twitcasting_task1/twitcasting.go
@@ -0,0 +1,140 @@
+package main
+
+import (
+ "database/sql"
+ "encoding/json"
+ "intimate"
+ "log"
+ "net/url"
+ "time"
+
+ "github.com/474420502/extractor"
+ "github.com/474420502/focus/compare"
+ "github.com/474420502/focus/tree/heap"
+ "github.com/474420502/requests"
+)
+
+// sstore 源存储实例, 为存储源数据的实现. 表格具体参考sql/intimate_source.sql
+var sstore *intimate.StoreSource = intimate.NewStoreSource(string(intimate.STTwitcasting))
+
+// estore 解析存储连接实例
+var estore *intimate.StoreExtractor = intimate.NewStoreExtractor()
+
+type SearchProfile struct {
+ UserName string `exp:".//span[@class='username']" method:"Text"`
+ UserId string // `exp:".//span[@class='fullname']" method:"Text"`
+ LiveUrl string `exp:".//div[@class='usertext']/a[@href]" method:"AttributeValue,href"`
+ Tag []string `exp:".//a[contains(@class, 'tag tag-mini')]" method:"Text"`
+ TagUrl []string `exp:".//a[contains(@class, 'tag tag-mini')]" method:"AttributeValue,href"`
+}
+
+func Execute() {
+ homeurl := "https://twitcasting.tv"
+ searchurl := "https://twitcasting.tv/rankingindex.php"
+ queuedict := make(map[string]bool)
+ queue := heap.New(compare.String)
+ queue.Put(searchurl)
+ queuedict[searchurl] = true
+ ses := requests.NewSession()
+ ses.Config().SetTimeout(15)
+
+ var surl interface{}
+ var ok bool
+
+ ps := intimate.NewPerfectShutdown()
+
+ for surl, ok = queue.Pop(); ok && !ps.IsClose(); surl, ok = queue.Pop() {
+ u, err := url.Parse(surl.(string))
+ if err != nil {
+ log.Println(err)
+ continue
+ }
+
+ resp, err := ses.Get(u.String()).Execute()
+ if err != nil {
+ log.Println(err)
+ log.Println(u.String(), surl)
+ continue
+ // log.Panic(err)
+ }
+
+ etor := extractor.ExtractXml(resp.Content())
+ result, err := etor.XPaths("//p[@class='taglist']/a[contains(@class, 'tag')]/@href")
+ if err != nil {
+ panic(err)
+ }
+
+ for _, href := range result.GetTexts() {
+
+ wurl := homeurl + href
+ if ok := queuedict[wurl]; !ok {
+ log.Println(wurl)
+ sl := &intimate.StreamerList{}
+ sl.Platform = intimate.Ptwitcasting
+ sl.Url = wurl
+ sl.Operator = 0
+ sl.UpdateInterval = 120
+ sl.UpdateTime = time.Now()
+
+ estore.InsertStreamerList(sl)
+
+ queue.Put(wurl)
+ queuedict[wurl] = true
+ }
+ }
+
+ xps, err := etor.XPaths("//div[@class='tw-search-result-row']")
+ if err != nil {
+ log.Println(surl, err)
+ continue
+ }
+
+ var splist = xps.ForEachTag(SearchProfile{})
+ for _, isp := range splist {
+ sp := isp.(*SearchProfile)
+ if sp.LiveUrl == "" {
+ continue
+ }
+
+ sp.UserId = sp.LiveUrl[1:]
+ for i := 0; i < len(sp.TagUrl); i++ {
+ wurl := homeurl + sp.TagUrl[i]
+ sp.TagUrl[i] = wurl
+ if ok := queuedict[wurl]; !ok {
+ sl := &intimate.StreamerList{}
+ sl.Platform = intimate.Ptwitcasting
+ sl.Url = wurl
+ sl.Operator = 0
+ sl.UpdateInterval = 120
+ sl.UpdateTime = time.Now()
+ estore.InsertStreamerList(sl)
+
+ queue.Put(wurl)
+ queuedict[wurl] = true
+ }
+ }
+ // log.Println(sp.(SearchProfile))
+ }
+
+ log.Println("find user:", len(splist))
+ for _, isp := range splist {
+ sp := isp.(*SearchProfile)
+ // log.Println(sp)
+ streamer := &intimate.Streamer{}
+ streamer.Platform = intimate.Ptwitcasting
+ streamer.LiveUrl = sql.NullString{String: sp.LiveUrl, Valid: true}
+ if btags, err := json.Marshal(sp.Tag); err != nil {
+ log.Println(err)
+ } else {
+ streamer.Tags = btags
+ }
+ streamer.UpdateInterval = 120
+ streamer.UpdateTime = sql.NullTime{Time: time.Now(), Valid: true}
+ streamer.UserName = sql.NullString{String: sp.UserName, Valid: true}
+ streamer.UserId = sp.UserId
+ estore.InsertStreamer(streamer)
+ }
+
+ log.Println("finish remain", queue.Size())
+ }
+}
diff --git a/tasks/twitch/twitch_task1/task_twitch.go b/tasks/twitch/twitch_task1/task_twitch.go
index 2b385ed..d98b28b 100644
--- a/tasks/twitch/twitch_task1/task_twitch.go
+++ b/tasks/twitch/twitch_task1/task_twitch.go
@@ -51,7 +51,6 @@ func (cl *ChannelLink) Execute() {
var hrefs map[string]bool = make(map[string]bool)
var delayerror = 5
- var samecount = 0
for i := 0; i <= 200; i++ {
cards, err := wd.FindElements(selenium.ByXPATH, "//span/a[contains(@data-a-target,'card-') and @href]")
if err != nil {
@@ -59,7 +58,7 @@ func (cl *ChannelLink) Execute() {
break
}
- if len(cards) == samecount {
+ if len(hrefs) == 0 {
delayerror--
if delayerror <= 0 {
break
@@ -80,7 +79,7 @@ func (cl *ChannelLink) Execute() {
}
break
}
- samecount = len(cards)
+
if ps.IsClose() {
break
}
diff --git a/tasks/twitch/twitch_task2/task_twitch.go b/tasks/twitch/twitch_task2/task_twitch.go
index 52773de..915a969 100644
--- a/tasks/twitch/twitch_task2/task_twitch.go
+++ b/tasks/twitch/twitch_task2/task_twitch.go
@@ -29,13 +29,16 @@ func (cl *UserList) Execute() {
//article//a[@data-a-target='preview-card-title-link']
wd := intimate.GetChromeDriver(3030)
+ defer wd.Close()
+ defer wd.Quit()
ps := intimate.NewPerfectShutdown()
counter := intimate.NewCounter()
counter.SetMaxLimit(100)
counter.SetMaxToDo(func(olist ...interface{}) error {
owd := olist[0].(*selenium.WebDriver)
- (*owd).Close()
- (*owd).Quit()
+ if err := (*owd).Quit(); err != nil {
+ log.Println(err)
+ }
*owd = intimate.GetChromeDriver(3030)
return nil
}, &wd)
diff --git a/utils.go b/utils.go
index ebee59b..cb7d782 100644
--- a/utils.go
+++ b/utils.go
@@ -34,6 +34,27 @@ func ParseNumber(number string) (int64, error) {
return strconv.ParseInt(number, 10, 64)
}
+// ParseNumberEx 解析带字符的数字
+func ParseNumberEx(num string) (float64, error) {
+ num = strings.Trim(num, " ")
+ last := num[len(num)-1]
+ factor := 1.0
+ switch {
+ case last == 'k' || last == 'K':
+ factor = 1000.0
+ num = num[0 : len(num)-1]
+ case last == 'm' || last == 'M':
+ factor = 1000000.0
+ num = num[0 : len(num)-1]
+ }
+ i, err := strconv.ParseFloat(num, 64)
+ if err != nil {
+ return 0, err
+ }
+
+ return i * factor, nil
+}
+
// ParseDuration time to duration eg: 1:40:00 -> time.Duration
func ParseDuration(dt string) (time.Duration, error) {
@@ -103,14 +124,12 @@ func GetChromeDriver(port int) selenium.WebDriver {
panic(err)
}
runtime.SetFinalizer(wd, func(obj interface{}) {
-
if err := obj.(selenium.WebDriver).Close(); err != nil {
log.Println(err)
}
if err := obj.(selenium.WebDriver).Quit(); err != nil {
log.Println(err)
}
-
})
wd.ExecuteScript("windows.navigator.webdriver = undefined", nil)
if err != nil {