Merge branch 'release/v0.5.4'

2.删除hunter包引用和使用

.gitignore

@@ -1,3 +1,4 @@
+bin
 *.log
 log
 screenlog.*
@@ -6,5 +7,7 @@ screenlog.*
 intimate
 *.gz
 debug.test
-
+myblock
+run.sh
+stop.sh
 

build.sh

@@ -7,13 +7,17 @@ src=`pwd`
 for path in `ls -d $source_tasks`
 do
     echo $path
-    cd $path && go build
+    projectname=${path##*/}
+    projectworkspace=$src/bin/$projectname
+    cd $path && mkdir $projectworkspace -p  && go build -o $projectworkspace/$projectname
     cd $src
 done
 
 for path in `ls -d $extractor_tasks`
 do
     echo $path
-    cd $path && go build
+    projectname=${path##*/}
+    projectworkspace=$src/bin/$projectname
+    cd $path && mkdir $projectworkspace -p  && go build -o $projectworkspace/$projectname
     cd $src
 done

config.go

@@ -17,7 +17,7 @@ func init() {
 	InitConfig.Load()
 	// storeOpenrec = NewStore()
 
-	log.SetFlags(log.Llongfile | log.Ldate)
+	log.SetFlags(log.Llongfile | log.Ltime)
 }
 
 // Config 配置

config.yaml

@@ -1,3 +1,3 @@
 database:
-  source_uri: "root:@tcp(127.0.0.1:4000)/intimate_source?parseTime=true&loc=Local"
-  extractor_uri: "root:@tcp(127.0.0.1:4000)/intimate_extractor?parseTime=true&loc=Local"
+  source_uri: "root:@tcp(127.0.0.1:4000)/intimate_source?parseTime=true&loc=Local&charset=utf8mb4&collation=utf8mb4_unicode_ci"
+  extractor_uri: "root:@tcp(127.0.0.1:4000)/intimate_extractor?parseTime=true&loc=Local&charset=utf8mb4&collation=utf8mb4_unicode_ci"

crx/myblock.pem

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDSG09DSvB03TOe
+eOmQwfiCIf0wa2WRB31ewxa6i/PRgEKeJSUvIsIuaECUer2ss+J3rwSS2lDpGuiw
+FnsVyZqKI/+Rcuc83YJGYg6OAzVMz6UL8YCWhXu3huTJ+V+a5iNereIC69ZERRJt
+nXlWqsq6HKya+6BP9sX9CI4GTHQrnWBysAxsswhdnnnRvu+GxglWafSIzuS6OizT
+1M1CmkZxNvDJhTSOR7SJlIYm2kM5/fIL53BdndF2IGAjfV1WV7AjwhTfun5cViEO
+i8niQUIMY4L0AiO9grFD1g1xIYkeuVBoLxOUBzPxJwQmb64gseb9Dvt0BKLRGoou
+SIOyE+KVAgMBAAECggEAI4b6J2kR0VUBEDwmVHO0K38HUstqNHSVgrNO0dLt8sAz
+I44o5DhGqPW4a9L4ZS5SrkWyKonPcic6buISRIwfPVoacjQBfVWAXJnil6lbtyYK
+ZMNcqLcgBRfCcpOgEq91DiKta6yIwekDFXVyCdFd78v+9ML1J+hUsLVkXJTLdP88
+PGamRWVd6vGy3QMRjyM29GLPgS+/6Vrp1cptSuYNqYhlszohmu8lBvzjH9jbPh9d
+GFrrd8Bs7IRCdtKZig/3fbln4JEyyOYE+gcT2jplPksB6mR/5DBIdkVbeuFwGB0+
+h1/PKlprNQt7+Ei0HhHnTib7lZP8WGo4HkSi7PsAGQKBgQD1Ptho0wJiI2+6gL1O
+iNsEJVKIQ2Sxdx3wI/qudphM99t6xKCpPyVI2Nd9PBf2jbZjGAaz+P/KQYxEqb6i
+PRcQ+i99wCQoRfnRvUbKA4goEpKwRXmvn+499dm6D5pEuumOXGQYCmaFXuLTRN/I
+BL6GNgLtoZAlLjUXaWtk8TszGQKBgQDbUf3p3HLpCjRvRDW/vA5xj+08t7xtF9uO
+NilGK79uOA4VnxE2w3ioYqQ7t3I8J/0rAzGKq3tylg4QX6UpQ4b2koRr2B3cqoAk
+dsRdNWAHwCNepz8hTLsZyuihzbNv2nHmoqhzjK/FcrBHx5NAM+T6OBpLzQBnbUzk
+3wIcqm223QKBgQDo/IRxyY0pGMtLXoT6ODACF0b6JzRhGG37tuKvngGAlbQQRP7w
+6wmL1F2cH1wQon7UU34CupqfVnhgvvZZgToJqfU2PTTcgeYc6Pl4b7SJhWOQTOCX
+BZQ7jvYCulHv27aIxaNd53uQVx2cYoFKr58lN+i+QtADUoujq0YYxshb+QKBgQDW
+ZOti7kZCeuBRGIu2V56C8uBFp5MBzf2polZsqx1iIFfcWPfZ4fGUIYFMgwKfvbOl
+lWSbmxB9LiSnaugoU0OezBG43rYqXV4Qxy0jtKagTPoGcFWtNrX7+7e3XD8Zi6Am
+hkFHW3MEAB5EvNq8Oz6OP8Os78SCVn2BimMlJJFF3QKBgQCF+aEAiBv+ivcmHUeP
+2eBq9nLltPFAfXJ/p31MMQ6Jgo36DBqUeoLeyq/WfIXvwqbVbP9fANZrKoTPbI97
+dilCHUoO33rafXJy6jtaggtpz14tt9soecTop0vM/rU7tGtfBe6NXg9LRl+oDJCU
+37I3a9Is+2CLyAUXWCk9mLfFsQ==
+-----END PRIVATE KEY-----

extractor/openrec_extractor/main.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"net/http"
 	_ "net/http/pprof"
 )
 
@@ -16,11 +15,5 @@ import (
 */
 
 func main() {
-
-	go func() {
-		http.ListenAndServe("0.0.0.0:8899", nil)
-	}()
-
-	oe := &OpenrecExtractor{}
-	oe.Execute()
+	Execute()
 }

extractor/openrec_extractor/openrec_extractor.go

@@ -3,57 +3,52 @@ package main
 import (
 	"database/sql"
 	"encoding/json"
+	"errors"
 	"intimate"
 	"log"
-	"os"
-	"os/signal"
-	"regexp"
-	"runtime"
-	"strconv"
-	"strings"
-	"sync/atomic"
-	"syscall"
 	"time"
 
+	"github.com/474420502/extractor"
 	"github.com/tidwall/gjson"
 )
 
-// OpenrecExtractor 提取方法
-type OpenrecExtractor struct {
-	user       *intimate.ExtractorSource
-	userLive   *intimate.ExtractorSource
-	supporters *intimate.ExtractorSource
+var estore = intimate.NewStoreExtractor()
+var sstore = intimate.NewStoreSource(string(intimate.STOpenrec))
+
+//UserInfo 提取信息的结构体
+type UserInfo struct {
+	UserName  string `exp:"//p[ contains(@class, 'c-global__user__profile__list__name__text')]"`
+	Followers int64  `exp:"//p[@class='c-global__user__count__row__right js-userCountFollowers']" mth:"r:ParseNumber"`
+	Views     int64  `exp:"//ul[@class='c-contents']//p[@class='c-thumbnailVideo__footer__liveCount']" mth:"r:ExtractNumber"`
 }
 
-func (oe *OpenrecExtractor) Execute() {
+//UserLive 提取信息的结构体
+type UserLive struct {
+	Title         string   `exp:"//h1[contains(@class,'MovieTitle__Title')]"`
+	LiveStartTime string   `exp:"//meta[@itemprop='uploadDate']/@content"`
+	LiveEndTime   string   `exp:"//meta[@itemprop='duration']/@content"`
+	Tags          []string `exp:"//a[contains(@class,'TagButton')]"`
+}
 
-	var loop int32 = 1
+// Execute 执行
+func Execute() {
 
-	go func() {
-		signalchan := make(chan os.Signal)
-		signal.Notify(signalchan, syscall.SIGKILL, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGSTOP)
-		log.Println("accept stop command:", <-signalchan)
-		atomic.StoreInt32(&loop, 0)
-	}()
+	ps := intimate.NewPerfectShutdown()
 
-	estore := intimate.NewStoreExtractor()
-	sstore := intimate.NewStoreSource(string(intimate.STOpenrec))
 	var lasterr error = nil
+	for !ps.IsClose() {
+		var err error
 
-	for atomic.LoadInt32(&loop) > 0 {
-
-		runtime.GC()
-		time.Sleep(time.Nanosecond)
-
-		source, err := sstore.Pop(string(intimate.TTOpenrecUser), 0)
+		source, err := sstore.Pop(intimate.TOpenrecUser, 0)
 		if err != nil {
 			if err != lasterr {
 				log.Println(err, lasterr)
 				lasterr = err
 			}
-			time.Sleep(time.Second * 2)
+			time.Sleep(time.Second * 5)
 			continue
 		}
+		lasterr = nil
 
 		sdata := source.Ext.([]byte)
 		datamap := gjson.ParseBytes(sdata).Map()
@@ -63,102 +58,30 @@ func (oe *OpenrecExtractor) Execute() {
 
 		streamer := &intimate.Streamer{}
 		streamer.UserId = userId
-		streamer.Platform = string(intimate.Popenrec)
+		// streamer.Platform = intimate.Popenrec 不需要更新字段
 
-		oe.user = intimate.NewExtractorSource(datamap["html_user"])
-		oe.user.CreateExtractor()
+		htmlUser := datamap["html_user"]
 
-		oe.userLive = intimate.NewExtractorSource(datamap["html_live"])
-		oe.userLive.CreateExtractor()
+		userEtor := extractor.ExtractHtmlString(htmlUser.String())
+		ui, ok1 := userEtor.GetObjectByTag(UserInfo{}).(*UserInfo)
 
-		oe.supporters = intimate.NewExtractorSource(datamap["json_supporters"])
+		htmlLive := datamap["html_live"]
+
+		liveEtor := extractor.ExtractHtmlString(htmlLive.String())
+		ul, ok2 := liveEtor.GetObjectByTag(UserLive{}).(*UserLive)
+
+		jsonSupporters := datamap["json_supporters"]
 		clog := &intimate.CollectLog{}
 
-		// log.Println(anchorId)
-
-		oe.extractFollowers(clog)
-		oe.extractUserName(streamer)
-		oe.extractViewsAndLiveStreaming(clog)
-		oe.extractGiversAndGratuity(clog)
-		oe.extractLive(clog)
-		oe.extractTags(clog)
-
-		streamer.Uid = source.StreamerId.Int64
-		streamer.UpdateTime = source.UpdateTime
-
-		clog.Platform = string(intimate.Popenrec)
-		clog.UserId = userId
-		clog.UpdateTime = source.UpdateTime
-
-		logUid := estore.InsertCollectLog(clog)
-
-		LiveUrl := "https://www.openrec.tv/live/" + userId
-		streamer.LiveUrl = sql.NullString{String: LiveUrl, Valid: true}
-		streamer.LatestLogUid = logUid
-		streamer.Operator = 0
-		estore.UpdateStreamer(streamer)
-
-		source.Operator = int32(intimate.OperatorExtractorOK)
-		sstore.UpdateOperator(source)
-
+		if ok1 {
+			clog.Followers = sql.NullInt64{Int64: ui.Followers, Valid: true}
+			clog.Views = sql.NullInt64{Int64: ui.Views, Valid: true}
+			if ui.Views != 0 {
+				clog.IsLiveStreaming = true
 			}
-}
+			streamer.UserName = sql.NullString{String: ui.UserName, Valid: true}
 
-func (oe *OpenrecExtractor) extractFollowers(clog intimate.ISet) {
-	extractor := oe.user.GetExtractor()
-	xp, err := extractor.XPathResult("//p[@class='c-global__user__count__row__right js-userCountFollowers']/text()")
-	if err != nil {
-		log.Println(err)
-	}
-	if !xp.NodeIter().Next() {
-		log.Println("不存在粉丝数")
-	}
-
-	followers := strings.ReplaceAll(xp.String(), ",", "")
-	followersInt, err := strconv.ParseInt(followers, 10, 64)
-	if err != nil {
-		log.Println(err)
-	}
-
-	clog.Set("Followers", sql.NullInt64{Int64: followersInt, Valid: true})
-}
-
-func (oe *OpenrecExtractor) extractUserName(streamer intimate.ISet) {
-	extractor := oe.user.GetExtractor()
-	xp, err := extractor.XPathResult("//p[ contains(@class, 'c-global__user__profile__list__name__text')]/text()")
-	if err != nil {
-		log.Println(err)
-	} else {
-		if xp.NodeIter().Next() {
-			userName := xp.String()
-			streamer.Set("UserName", sql.NullString{String: userName, Valid: true})
-		}
-	}
-}
-
-func (oe *OpenrecExtractor) extractViewsAndLiveStreaming(clog intimate.ISet) {
-	extractor := oe.user.GetExtractor()
-	// c-contents
-	xp, err := extractor.XPathResult("//ul[@class='c-contents']//p[@class='c-thumbnailVideo__footer__liveCount']/text()")
-	if err != nil {
-		log.Println(err)
-	}
-	if xp.NodeIter().Next() {
-		views := regexp.MustCompile(`[0-9,]+`).FindString(xp.String())
-		views = strings.ReplaceAll(views, ",", "")
-		viewsint, err := strconv.Atoi(views)
-		if err != nil {
-			log.Println(err)
-		}
-
-		clog.Set("Views", sql.NullInt64{Int64: int64(viewsint), Valid: true})
-		clog.Set("IsLiveStreaming", true)
-	}
-}
-
-func (oe *OpenrecExtractor) extractGiversAndGratuity(clog intimate.ISet) {
-	// extractor := oe.user.GetExtractor()
-	giverjson := oe.supporters.GetSource()
+			giverjson := jsonSupporters
 			var givers []interface{}
 			var gratuity int64 = 0
 
@@ -173,64 +96,72 @@ func (oe *OpenrecExtractor) extractGiversAndGratuity(clog intimate.ISet) {
 			giversbytes, err := json.Marshal(givers)
 			if err != nil {
 				log.Println(err)
-		clog.Set("ErrorMsg", sql.NullString{String: err.Error(), Valid: true})
+				clog.ErrorMsg = sql.NullString{String: err.Error(), Valid: true}
 			} else {
-		clog.Set("Giver", giversbytes)
+				clog.Giver = giversbytes
+			}
+
+			clog.Gratuity = sql.NullInt64{Int64: gratuity, Valid: true}
+		} else {
+			log.Println("UserInfo may be not exists")
+			estore.UpdateError(streamer, errors.New("UserInfo may be not exists"))
+			continue
+		}
+
+		//log.Println(ul)
+		if ok2 {
+			clog.LiveTitle = sql.NullString{String: ul.Title, Valid: true}
+
+			startTime, err := time.ParseInLocation("2006-01-02T15:04:05Z07:00", ul.LiveStartTime, time.Local)
+			if err != nil {
+				log.Println(err)
+			} else {
+				clog.LiveStartTime = sql.NullTime{Time: startTime.Local(), Valid: true}
+				duration, err := intimate.ParseDuration(ul.LiveEndTime)
+				if err != nil {
+					log.Println(err)
+				} else {
+					endTime := startTime.Add(duration)
+					clog.LiveStartTime = sql.NullTime{Time: endTime.Local(), Valid: true}
+				}
+			}
+
+			if tags, err := json.Marshal(ul.Tags); err == nil {
+				clog.Tags = tags
+			} else {
+				log.Println("json error", ul.Tags, clog.Tags)
+			}
+		}
+
+		streamer.Uid = source.StreamerId.Int64
+		streamer.UpdateTime = source.UpdateTime
+		streamer.Tags = clog.Tags
+
+		clog.Platform = intimate.Popenrec
+		clog.UserId = userId
+		clog.UpdateTime = source.UpdateTime
+		clog.StreamerUid = streamer.Uid
+
+		logUid := estore.InsertClog(clog)
+
+		LiveUrl := "https://www.openrec.tv/live/" + userId
+
+		streamer.LiveUrl = sql.NullString{String: LiveUrl, Valid: true}
+		streamer.LatestLogUid = logUid
+		// streamer.Operator = 0
+
+		log.Println(streamer.UserId)
+		estore.Update(streamer,
+			"user_name", streamer.UserName,
+			"user_id", streamer.UserId,
+			"live_url", streamer.LiveUrl,
+			"latest_log_uid", streamer.LatestLogUid,
+			"update_time", streamer.UpdateTime,
+			"tags", streamer.Tags,
+		)
+
+		source.Operator = int32(intimate.OperatorExtractorOK)
+		sstore.UpdateOperator(source)
 	}
 
-	clog.Set("Gratuity", sql.NullInt64{Int64: gratuity, Valid: true})
-}
-
-func (oe *OpenrecExtractor) extractLive(clog intimate.ISet) {
-	extractor := oe.userLive.GetExtractor()
-	mathes := regexp.MustCompile("MovieTitle__Title[^>]+>(.{1,50})</h1>").FindStringSubmatch(oe.userLive.GetSource().Str)
-	if len(mathes) == 2 {
-
-		clog.Set("LiveTitle", sql.NullString{String: mathes[1], Valid: true})
-
-		content, err := extractor.XPathResult("//meta[@itemprop='uploadDate']/@content")
-		if err != nil {
-			log.Println(err)
-		}
-
-		iter := content.NodeIter()
-		if iter.Next() {
-			tm, err := time.ParseInLocation("2006-01-02T15:04:05Z07:00", iter.Node().NodeValue(), time.Local)
-			if err != nil {
-				log.Println(err)
-			}
-			// log.Println(iter.Node().NodeValue(), tm.Local())
-			clog.Set("LiveStartTime", sql.NullTime{Time: tm.Local(), Valid: true})
-
-			duration, err := extractor.XPathResult("//meta[@itemprop='duration']/@content")
-			if err != nil {
-				log.Println(err)
-			}
-
-			diter := duration.NodeIter()
-			if diter.Next() {
-
-				dt, err := intimate.ParseDuration(diter.Node().NodeValue())
-				if err != nil {
-					log.Println(err)
-				}
-				endtm := tm.Add(dt)
-				clog.Set("LiveEndTime", sql.NullTime{Time: endtm.Local(), Valid: true})
-			}
-		}
-	}
-}
-
-func (oe *OpenrecExtractor) extractTags(clog intimate.ISet) {
-	var tags []string
-	matheslist := regexp.MustCompile(`<[^>]+TagButton[^>]+>([^<]{1,100})<`).FindAllStringSubmatch(oe.userLive.GetSource().Str, -1)
-	for _, m := range matheslist {
-		tags = append(tags, m[1])
-	}
-	tagsBytes, err := json.Marshal(tags)
-	if err != nil {
-		log.Println(err)
-	}
-
-	clog.Set("Tags", tagsBytes)
 }

extractor/openrec_extractor/openrec_test.go

@@ -7,7 +7,6 @@ import (
 	"testing"
 	"time"
 
-	"github.com/474420502/hunter"
 	"github.com/lestrrat-go/libxml2"
 )
 
@@ -91,29 +90,6 @@ func TestCase(t *testing.T) {
 	t.Error(xr)
 }
 
-func TestUserName(t *testing.T) {
-	f, err := os.Open("test.html")
-	if err != nil {
-		panic(err)
-	}
-	data, err := ioutil.ReadAll(f)
-	if err != nil {
-		panic(err)
-	}
-
-	extractor := hunter.NewExtractor(data)
-	xp, err := extractor.XPathResult("//p[ contains(@class, 'c-global__user__profile__list__name__text')]/text()")
-	if err != nil {
-		t.Error(err)
-	} else {
-		if xp.NodeIter().Next() {
-			userName := xp.String()
-			t.Error(userName)
-		}
-	}
-}
-
 func TestExtractor(t *testing.T) {
-	oe := &OpenrecExtractor{}
-	oe.Execute()
+	Execute()
 }

extractor/twitcasting_extractor/.gitignore

@@ -0,0 +1,4 @@
+*.html
+log
+screenlog.*
+twitcasting_extractor

extractor/twitcasting_extractor/twitcasting_extractor.go

@@ -0,0 +1,152 @@
+package main
+
+import (
+	"database/sql"
+	"intimate"
+	"log"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/474420502/extractor"
+	"github.com/474420502/requests"
+)
+
+// sstore  源存储实例, 为存储源数据的实现. 表格具体参考sql/intimate_source.sql
+var sstore *intimate.StoreSource = intimate.NewStoreSource(string(intimate.STOpenrec))
+
+// estore 解析存储连接实例
+var estore *intimate.StoreExtractor = intimate.NewStoreExtractor()
+
+type LiveData struct {
+	UserName     string   `exp:"//span[@class='tw-live-author__info-username']" method:"Text"`
+	Follower     string   `exp:"(//span[@class='tw-user-nav-list-count'])[2]" method:"Text"`
+	MaxViews     string   `exp:"//span[@id='max_viewer_count']" method:"Text"`
+	LiveTitle    string   `exp:"//meta[@property='og:title']" method:"AttributeValue,content"`
+	LiveStart    string   `exp:"//time[@data-kind='relative']" method:"AttributeValue,datetime"`
+	LiveDuration string   `exp:"//span[@id='updatetimer']" method:"AttributeValue,data-duration"`
+	Tags         []string `exp:"//div[@class='tw-live-author__commandbox--tags']//a[@class='tag  tag-info']" method:"Text"`
+}
+
+func main() {
+
+	ps := intimate.NewPerfectShutdown()
+	ses := requests.NewSession()
+
+	for !ps.IsClose() {
+
+		streamer, err := estore.Pop(intimate.Ptwitcasting)
+		if err != nil {
+			log.Println(err, streamer)
+		}
+
+		streamer.LiveUrl = sql.NullString{String: "https://twitcasting.tv/" + streamer.UserId, Valid: true}
+		resp, err := ses.Get(streamer.LiveUrl.String).Execute()
+		if err != nil {
+			estore.UpdateError(streamer, err)
+			log.Println(err, streamer.UserId)
+			continue
+		}
+		var ldata *LiveData
+		etor := extractor.ExtractHtml(resp.Content())
+		ldata = etor.GetObjectByTag(LiveData{}).(*LiveData)
+		ldata.MaxViews = regexp.MustCompile("\\d+").FindString(ldata.MaxViews)
+		coincount := 0
+
+		for i := 0; ; i++ {
+
+			giverurl := streamer.LiveUrl.String + "/backers/" + strconv.Itoa(i)
+			resp, err = ses.Get(giverurl).Execute()
+			if err != nil {
+				estore.UpdateError(streamer, err)
+				log.Panic(err)
+			}
+
+			etor := extractor.ExtractHtml(resp.Content())
+			xp, err := etor.XPaths("//td[@class='tw-memorial-table-recent-point']")
+			if err != nil {
+				estore.UpdateError(streamer, err)
+				log.Panic(err)
+			}
+
+			coins := xp.GetTexts()
+			for _, cointxt := range coins {
+				scointxt := strings.Split(cointxt, "/")
+				if len(scointxt) == 2 {
+					coin := strings.Trim(scointxt[1], " ")
+					c, err := strconv.Atoi(coin)
+					if err == nil {
+						coincount += c
+					}
+					// log.Println(coin, coincount)
+				} else {
+					log.Println("coin error: ", cointxt)
+				}
+			}
+
+			if len(coins) < 20 {
+				break
+			}
+		}
+
+		streamer.Platform = intimate.Ptwitcasting
+		streamer.UpdateTime = sql.NullTime{Time: time.Now(), Valid: true}
+		streamer.UserName = sql.NullString{String: ldata.UserName, Valid: true}
+		streamer.Operator = 0
+		// streamer.UpdateInterval = 60
+		clog := &intimate.CollectLog{}
+		clog.UserId = streamer.UserId
+		clog.Gratuity = sql.NullInt64{Int64: int64(coincount), Valid: true}
+		clog.Platform = streamer.Platform
+		clog.UpdateTime = streamer.UpdateTime
+		clog.LiveTitle = sql.NullString{String: ldata.LiveTitle, Valid: true}
+		fl, err := intimate.ParseNumberEx(ldata.Follower)
+		if err == nil {
+			clog.Followers = sql.NullInt64{Int64: int64(fl), Valid: true}
+			switch {
+			case fl <= 100:
+				streamer.UpdateInterval = 360
+			case fl <= 1000:
+				streamer.UpdateInterval = 240
+			case fl <= 100:
+				streamer.UpdateInterval = 120
+			default:
+				streamer.UpdateInterval = 60
+			}
+		} else {
+			log.Println(err)
+		}
+
+		views, err := strconv.Atoi(ldata.MaxViews)
+		if err == nil {
+			clog.Views = sql.NullInt64{Int64: int64(views), Valid: true}
+		} else {
+			clog.Views = sql.NullInt64{Int64: int64(0), Valid: true}
+			// log.Println(err, streamer.UserId)
+		}
+
+		// st, err := strconv.Atoi(ldata.LiveStart)
+		st, err := time.Parse("Mon, 02 Jan 2006 15:04:05 -0700", ldata.LiveStart)
+		if err == nil {
+			startTime := st
+			clog.LiveStartTime = sql.NullTime{Time: startTime, Valid: true}
+			dt, err := strconv.Atoi(ldata.LiveDuration)
+			if time.Now().Sub(startTime) >= time.Hour*24*90 {
+				streamer.Operator = 5
+			}
+
+			if err == nil {
+				endTime := startTime.Add((time.Duration)(dt) * time.Millisecond)
+				clog.LiveEndTime = sql.NullTime{Time: endTime, Valid: true}
+			} else {
+				log.Println(err, streamer.UserId)
+			}
+		} else {
+			log.Println(err, streamer.UserId)
+		}
+
+		streamer.LatestLogUid = estore.InsertClog(clog)
+		estore.UpdateStreamer(streamer)
+	}
+}

extractor/twitcasting_extractor/twitcasting_extractor_test.go

@@ -0,0 +1,23 @@
+package main
+
+import (
+	"testing"
+	"time"
+)
+
+// type LiveData struct {
+// 	UserName string `exp:".//span[@class='tw-live-author__info-username']" method:"Text"`
+// 	Follower string `exp:".//span[@class='tw-user-nav-list-count']" method:"Text"`
+// }
+
+func estMain(t *testing.T) {
+	main()
+}
+
+func TestDateFormat(t *testing.T) {
+	df := "Sat, 09 Sep 2017 18:19:17 +0900"
+
+	if _, err := time.Parse("Mon, 02 Jan 2006 15:04:05 -0700", df); err != nil {
+		t.Error(err)
+	}
+}

extractor/twitch_extractor/.gitignore

@@ -0,0 +1,4 @@
+*.html
+log
+screenlog.*
+twitch_extractor

extractor/twitch_extractor/tiwtch_extractor.go

@@ -0,0 +1,256 @@
+package main
+
+import (
+	"database/sql"
+	"encoding/json"
+	"intimate"
+	"log"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/tebeka/selenium"
+)
+
+// sstore  源存储实例, 为存储源数据的实现. 表格具体参考sql/intimate_source.sql
+var sstore *intimate.StoreSource = intimate.NewStoreSource(string(intimate.STTwitch))
+
+// estore 解析存储连接实例
+var estore *intimate.StoreExtractor = intimate.NewStoreExtractor()
+
+func main() {
+	wd := intimate.GetChromeDriver(3030)
+	ps := intimate.NewPerfectShutdown()
+
+	counter := intimate.NewCounter()
+	counter.SetMaxLimit(200)
+	counter.SetMaxToDo(func(olist ...interface{}) error {
+		owd := olist[0].(*selenium.WebDriver)
+		(*owd).Close()
+		(*owd).Quit()
+		*owd = intimate.GetChromeDriver(3030)
+		return nil
+	}, &wd)
+
+	var lasterr error = nil
+	// var err error
+
+	for !ps.IsClose() {
+		streamer, err := estore.Pop(intimate.Ptwitch, 0)
+		if streamer == nil || err != nil {
+			if err != lasterr {
+				log.Println(err, lasterr)
+				lasterr = err
+			}
+			time.Sleep(time.Second * 2)
+			continue
+		}
+
+		var updateUrl map[string]string
+		json.Unmarshal(streamer.UpdateUrl.([]byte), &updateUrl)
+		liveUrl := updateUrl["live"]
+		liveUrl = strings.Replace(liveUrl, "/watchparty", "", -1)
+		log.Println(liveUrl)
+
+		// err = wd.Get("https://www.twitch.tv/zoe_0601" + "/about")
+		err = wd.Get(liveUrl + "/about")
+		if err != nil {
+			log.Println(err)
+			estore.UpdateError(streamer, err)
+			time.Sleep(time.Second * 5)
+			continue
+		}
+
+		streamer.LiveUrl = sql.NullString{String: liveUrl, Valid: true}
+		clog := &intimate.CollectLog{}
+		clog.UserId = streamer.UserId
+		clog.Gratuity = sql.NullInt64{Int64: 0, Valid: false}
+
+		time.Sleep(time.Millisecond * 500)
+		err = extractUserName(wd, streamer)
+		if err != nil {
+			_, err = wd.FindElement(selenium.ByXPATH, "//a[@data-a-target='browse-channels-button']")
+			if err == nil {
+				log.Println(streamer.UserId, "may be cancell")
+				streamer.Operator = 5
+				streamer.UpdateTime = sql.NullTime{Time: time.Now(), Valid: true}
+				estore.UpdateStreamer(streamer)
+			}
+			continue
+		}
+		err = extractFollowers(wd, clog)
+		if err != nil {
+			continue
+		}
+
+		err = extractViews(wd, clog) // views + tags + gratuity
+		if err != nil {
+			// 不直播时提取礼物 gratuity
+			wd.WaitWithTimeout(func(web selenium.WebDriver) (bool, error) {
+				channelchat, err := wd.FindElement(selenium.ByXPATH, `//a[@data-a-target="channel-home-tab-Chat"]`)
+				btn, _ := web.FindElement(selenium.ByXPATH, `//button[@data-test-selector="expand-grabber"]`)
+				if (err == nil && channelchat != nil) || btn != nil {
+					if channelchat != nil {
+						channelchat.Click()
+					}
+					time.Sleep(time.Second)
+					extractGratuity(wd, clog)
+					return true, nil
+				}
+				return false, nil
+
+			}, time.Second*4)
+		}
+
+		streamer.Platform = intimate.Ptwitch
+		clog.Platform = streamer.Platform
+		clog.UpdateTime = sql.NullTime{Time: time.Now(), Valid: true}
+		lastClogId := estore.InsertClog(clog)
+
+		streamer.Operator = 10
+		streamer.LatestLogUid = lastClogId
+		if clog.Tags != nil {
+			streamer.Tags = clog.Tags
+		}
+
+		switch fl := clog.Followers.Int64; {
+		case fl > 100000:
+			streamer.UpdateInterval = 120
+		case fl > 10000:
+			streamer.UpdateInterval = 240
+		case fl > 1000:
+			streamer.UpdateInterval = 360
+		case fl > 100:
+			streamer.UpdateInterval = 720
+		case fl > 0:
+			streamer.UpdateInterval = 1440
+		}
+
+		streamer.UpdateTime = clog.UpdateTime
+		estore.UpdateStreamer(streamer)
+		counter.AddWithReset(1)
+	}
+
+	wd.Close()
+	wd.Quit()
+}
+
+func extractUserName(wd selenium.WebDriver, streamer *intimate.Streamer) error {
+	return wd.WaitWithTimeout(func(web selenium.WebDriver) (bool, error) {
+		label, err := web.FindElement(selenium.ByXPATH, "//a[@class='tw-interactive']//h1")
+		if err == nil {
+			if ltxt, err := label.Text(); err == nil && ltxt != "" {
+				// log.Println("label:", ltxt)
+				streamer.UserName = sql.NullString{String: ltxt, Valid: true}
+				return true, nil
+			}
+		}
+		return false, err
+	}, 15*time.Second)
+}
+
+func extractFollowers(wd selenium.WebDriver, clog *intimate.CollectLog) error {
+	return wd.WaitWithTimeout(func(web selenium.WebDriver) (bool, error) {
+		efollowers, err := web.FindElement(selenium.ByXPATH, "//div[@data-a-target='about-panel']//div[@class='tw-align-center']")
+		if err != nil {
+			return false, err
+		}
+		followers, err := efollowers.Text()
+		if err != nil || followers == "" {
+			return false, err
+		}
+		followers = regexp.MustCompile(`[\d,]+`).FindString(followers)
+		fint, _ := intimate.ParseNumber(followers)
+		clog.Followers = sql.NullInt64{Int64: int64(fint), Valid: true}
+		// log.Println("followers: ", followers, fint)
+		return true, nil
+	}, 4*time.Second)
+}
+
+func extractViews(wd selenium.WebDriver, clog *intimate.CollectLog) error {
+	return wd.WaitWithTimeout(func(web selenium.WebDriver) (bool, error) {
+		views, err := web.FindElement(selenium.ByXPATH, "//a[@data-a-target='home-live-overlay-button']/span")
+		if views != nil {
+			if txt, err := views.Text(); err == nil {
+
+				vint, _ := intimate.ParseNumber(txt)
+				clog.Views = sql.NullInt64{Int64: vint, Valid: true}
+				// log.Println("views:", txt)
+				views.Click()
+
+				extractTags(wd, clog)
+				extractTitle(wd, clog)
+				extractGratuity(wd, clog)
+
+				return true, nil
+			}
+		}
+		return false, err
+	}, time.Second*4)
+}
+
+func extractTitle(wd selenium.WebDriver, clog *intimate.CollectLog) error {
+	return wd.WaitWithTimeout(func(web selenium.WebDriver) (bool, error) {
+		title, err := web.FindElement(selenium.ByXPATH, `//h2[@data-a-target='stream-title']`)
+		if err == nil {
+			if txt, err := title.Text(); err == nil {
+				clog.LiveTitle = sql.NullString{String: txt, Valid: true}
+				return true, nil
+			}
+		}
+		return false, err
+	}, time.Second*4)
+}
+
+func extractTags(wd selenium.WebDriver, clog *intimate.CollectLog) error {
+	return wd.WaitWithTimeout(func(web selenium.WebDriver) (bool, error) {
+		tags, err := web.FindElements(selenium.ByXPATH, "//a[@aria-label and @data-a-target and @href]/div[@class and text()]")
+		if len(tags) == 0 {
+			return false, err
+		}
+
+		var stags []string
+		for _, tag := range tags {
+			if txt, err := tag.Text(); err == nil {
+				stags = append(stags, txt)
+			} else {
+				log.Println(err)
+			}
+		}
+		if len(stags) > 0 {
+			if tagbuf, err := json.Marshal(stags); err == nil {
+				clog.Tags = tagbuf
+			} else {
+				log.Println(err)
+			}
+		}
+
+		return true, nil
+	}, time.Second*4)
+}
+
+func extractGratuity(wd selenium.WebDriver, clog *intimate.CollectLog) error {
+	return wd.WaitWithTimeout(func(web selenium.WebDriver) (bool, error) {
+		btn, err := web.FindElement(selenium.ByXPATH, `//button[@data-test-selector="expand-grabber"]`)
+		if err == nil {
+			btn.Click()
+			time.Sleep(time.Second)
+			gifcount, err := web.FindElements(selenium.ByXPATH, `//div[@class="sub-gift-count tw-flex"]/p`)
+			if err == nil {
+				var gratuity int64 = 0
+				for _, gc := range gifcount {
+					if gtxt, err := gc.Text(); err == nil {
+						gint, _ := intimate.ParseNumber(gtxt)
+						gratuity += gint
+					} else {
+						log.Println(err)
+					}
+				}
+				clog.Gratuity = sql.NullInt64{Int64: gratuity, Valid: true}
+			}
+			return true, nil
+		}
+
+		return false, err
+	}, time.Second*4)
+}

extractor/twitch_extractor/twitch_test.go

@@ -0,0 +1,9 @@
+package main
+
+import (
+	"testing"
+)
+
+func TestCase0(t *testing.T) {
+	main()
+}

extractor_field.go

@@ -3,27 +3,54 @@ package intimate
 import (
 	"database/sql"
 	"reflect"
-
-	"github.com/474420502/hunter"
-	"github.com/tidwall/gjson"
+	"time"
 )
 
 type GetSet struct {
 }
 
+type StreamerList struct {
+	UrlHash  []byte   //
+	Platform Platform //
+	Url      string   //
+
+	Label sql.NullString //
+
+	Serialize interface{}
+
+	UpdateInterval int32
+	UpdateTime     time.Time //
+
+	ErrorMsg sql.NullString
+	Operator int32
+
+	LastOperator int32
+}
+
+// Get Simple Value
+func (sl *StreamerList) Get(field string) interface{} {
+	return reflect.ValueOf(sl).Elem().FieldByName(field).Interface()
+}
+
+// Set Simple Value
+func (sl *StreamerList) Set(field string, value interface{}) {
+	reflect.ValueOf(sl).Elem().FieldByName(field).Set(reflect.ValueOf(value))
+}
+
 type Streamer struct {
 	Uid      int64    //
-	Platform string //
+	Platform Platform //
 	UserId   string   //
 
 	UserName sql.NullString //
 	LiveUrl  sql.NullString //
 	Channel  sql.NullString //
+	Tags     interface{}
 	Ext      interface{} //
 
 	IsUpdateStreamer bool // 更新上面的内容
 	IsUpdateUrl      bool
-	updateInterval   int32
+	UpdateInterval   int32
 	UpdateUrl        interface{}
 	LatestLogUid     int64
 	UpdateTime       sql.NullTime //
@@ -48,7 +75,7 @@ type CollectLog struct {
 	LogUid      int64 // 日志id
 	StreamerUid int64 // StreamerId 表id与
 
-	Platform        string         //
+	Platform        Platform       //
 	UserId          string         // 平台的UserId
 	IsLiveStreaming bool           //
 	IsError         bool           //
@@ -74,26 +101,3 @@ func (cl *CollectLog) Get(field string) interface{} {
 func (cl *CollectLog) Set(field string, value interface{}) {
 	reflect.ValueOf(cl).Elem().FieldByName(field).Set(reflect.ValueOf(value))
 }
-
-type ExtractorSource struct {
-	source    gjson.Result
-	extractor *hunter.Extractor
-}
-
-func NewExtractorSource(gr gjson.Result) *ExtractorSource {
-	es := &ExtractorSource{}
-	es.source = gr
-	return es
-}
-
-func (es *ExtractorSource) CreateExtractor() {
-	es.extractor = hunter.NewExtractor([]byte(es.source.Str))
-}
-
-func (es *ExtractorSource) GetSource() gjson.Result {
-	return es.source
-}
-
-func (es *ExtractorSource) GetExtractor() *hunter.Extractor {
-	return es.extractor
-}

go.mod

@@ -1,12 +1,15 @@
 module intimate
 
-go 1.14
+go 1.15
 
 require (
-	github.com/474420502/gcurl v0.1.2
-	github.com/474420502/hunter v0.3.0
+	github.com/474420502/extractor v0.9.5
+	github.com/474420502/focus v0.12.0
+	github.com/474420502/gcurl v0.2.0
+	github.com/474420502/requests v1.7.0
 	github.com/go-sql-driver/mysql v1.5.0
 	github.com/lestrrat-go/libxml2 v0.0.0-20200215080510-6483566f52cb
+	github.com/tebeka/selenium v0.9.9
 	github.com/tidwall/gjson v1.6.0
 	github.com/tidwall/pretty v1.0.1 // indirect
 	golang.org/x/net v0.0.0-20200707034311-ab3426394381 // indirect

go.sum

@@ -2,14 +2,20 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
 cloud.google.com/go v0.41.0/go.mod h1:OauMR7DV8fzvZIl2qg6rkaIhD/vmgk4iwEw/h6ercmg=
+github.com/474420502/extractor v0.9.5 h1:nM3/1tNL8BHS3PN9eXvm8Ve3hqTReKNB+ba8VpdL8bw=
+github.com/474420502/extractor v0.9.5/go.mod h1:vJnXWmvO5bJDW4Yag0GoE2GxtHRg03TAxp2oXN1DcSY=
 github.com/474420502/focus v0.12.0 h1:+icbmj7IEOefvTegHt5EpcHt6WFbe2miIrceUJx2Evo=
 github.com/474420502/focus v0.12.0/go.mod h1:d0PMjtMxFz1a9HIhwyFPkWa+JF+0LgOrEUfd8iZka6s=
 github.com/474420502/gcurl v0.1.2 h1:ON9Yz3IgAdtDlFlHfkAJ3aIEBDxH0RiViPE5ST5ohKg=
 github.com/474420502/gcurl v0.1.2/go.mod h1:hws5q/Ao64bXLLDnldz9VyTQUndTWc/i5DzdEazFfoM=
-github.com/474420502/hunter v0.3.0 h1:0VPi1MInxjHOta3da4v0ALWK0y3/X4/6nUSLFvdbiFU=
-github.com/474420502/hunter v0.3.0/go.mod h1:pe4Xr/I+2agvq339vS/OZV+EiHAWtpXQs75rioSW9oA=
+github.com/474420502/gcurl v0.2.0 h1:m6+vw4NX4f5Tfp7c3nuaIgHUE/7zTX6K3xK+pTCBoCo=
+github.com/474420502/gcurl v0.2.0/go.mod h1:kJZDbgXn5wbAaR+hhBi4Sbw44P4igJ7qYXC6mejLuhQ=
+github.com/474420502/htmlquery v1.2.4-0.20200812072201-e871dd09247a h1:E1T6CYQKsUn7fMvNbeKfISjBLfOJjZX4KpWwStT20Kc=
+github.com/474420502/htmlquery v1.2.4-0.20200812072201-e871dd09247a/go.mod h1:AoSN890esHwNKecV0tCs+W0ele1xgFL1Jqk6UcrdxgU=
 github.com/474420502/requests v1.6.0 h1:f4h4j40eT0P5whhg9LdkotD8CaKjtuDu/vz9iSUkCgY=
 github.com/474420502/requests v1.6.0/go.mod h1:SLXrQ5dL9c7dkIeKNUCBAjOIt3J9KFCS2RQjWJecNwo=
+github.com/474420502/requests v1.7.0 h1:oaBwVrxZ7yZ+hDOKwHm2NflYib2y1geIUxBxQ2U48mw=
+github.com/474420502/requests v1.7.0/go.mod h1:SLXrQ5dL9c7dkIeKNUCBAjOIt3J9KFCS2RQjWJecNwo=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802 h1:1BDTz0u9nC3//pOCMdNH+CiXJVYJh5UQNCOBG7jbELc=
@@ -20,6 +26,8 @@ github.com/Pallinder/go-randomdata v1.1.0 h1:gUubB1IEUliFmzjqjhf+bgkg1o6uoFIkRsP
 github.com/Pallinder/go-randomdata v1.1.0/go.mod h1:yHmJgulpD2Nfrm0cR9tI/+oAgRqCQQixsA8HyRZfV9Y=
 github.com/Pallinder/go-randomdata v1.2.0 h1:DZ41wBchNRb/0GfsePLiSwb0PHZmT67XY00lCDlaYPg=
 github.com/Pallinder/go-randomdata v1.2.0/go.mod h1:yHmJgulpD2Nfrm0cR9tI/+oAgRqCQQixsA8HyRZfV9Y=
+github.com/antchfx/xpath v1.1.6 h1:6sVh6hB5T6phw1pFpHRQ+C4bd8sNI+O58flqtg7h0R0=
+github.com/antchfx/xpath v1.1.6/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
 github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
@@ -36,6 +44,8 @@ github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gG
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
@@ -66,6 +76,8 @@ github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYe
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/tebeka/selenium v0.9.9 h1:cNziB+etNgyH/7KlNI7RMC1ua5aH1+5wUlFQyzeMh+w=
 github.com/tebeka/selenium v0.9.9/go.mod h1:5Fr8+pUvU6B1OiPfkdCKdXZyr5znvVkxuPd0NOdZCQc=
 github.com/tidwall/gjson v1.3.2/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls=
@@ -103,6 +115,7 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k=
 golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -162,6 +175,8 @@ gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a h1:LJwr7TCTghdatWv40WobzlKXc9c4s8oGa7QKJUtHhWA=

platform_list.go

@@ -4,6 +4,12 @@ package intimate
 type Platform string
 
 const (
-	// Popenrec openrec源table名称
+	// Popenrec openrec 平台
 	Popenrec Platform = "openrec"
+
+	// Ptwitch twitch 平台
+	Ptwitch Platform = "twitch"
+
+	// Ptwitcasting twitcasting 平台
+	Ptwitcasting Platform = "twitcasting"
 )

source_field.go

@@ -19,7 +19,7 @@ type Source struct {
 	UpdateTime sql.NullTime   //
 	ErrorMsg   sql.NullString //
 
-	TargetType string //
+	Target   Target //
 	Operator int32  //
 
 	LastOperator int32

sql/intimate_extractor.sql

@@ -1,6 +1,25 @@
 create database if not exists `intimate_extractor`;
 use intimate_extractor;
 
+CREATE TABLE IF NOT EXISTS `streamer_list` ( 
+  `urlhash`         varchar(32) NOT NULL COMMENT '平台',
+  `url`             text COMMENT 'url获取streamer列表的url', 
+  `platform`        varchar(255) NOT NULL COMMENT '平台',
+  `label`           varchar(255) DEFAULT NULL COMMENT '必须的时候打上标签',
+  `serialize`       blob DEFAULT NULL COMMENT '保存进程的必要计算数据',
+
+  `update_interval` int  DEFAULT 120 COMMENT '分钟单位, 默认120分钟, 下次更新的时间间隔',
+  `update_time`     Timestamp   NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间',
+
+  `error_msg` text DEFAULT NULL COMMENT '错误信息',
+  `operator`      int DEFAULT 0 COMMENT '操作标志位, 根据不同解析方法有不同标志',
+
+  PRIMARY KEY (`urlhash`),
+  KEY `platform_idx` (`platform`),
+  KEY `update_time_idx` (`update_time`),
+  KEY `operator_idx` (`operator`)
+)
+
 CREATE TABLE IF NOT EXISTS `streamer` (
   `uid`           bigint AUTO_INCREMENT COMMENT '自增UID, 便于查询定位',
   `platform`      varchar(255) NOT NULL COMMENT '平台',
@@ -8,6 +27,7 @@ CREATE TABLE IF NOT EXISTS `streamer` (
   `user_name`   varchar(255) DEFAULT NULL COMMENT '用户名字 区别于ID',
   `live_url`      text COMMENT '直播的url', 
   `channel`       varchar(128) DEFAULT NULL COMMENT'所属 频道,分类 未必所有平台都有明确的标签',
+  `tag`           json DEFAULT NULL COMMENT 'streamer 最新的tag',
   `ext`           json DEFAULT NULL COMMENT '扩展类型, 把一些可能需要但是没字段的数据放在json扩展',
 
   `is_update_streamer` tinyint(1) DEFAULT 0 COMMENT '是否需要持续更新streamer的信息. 1为需要,0则否',

sql/intimate_source.sql

@@ -19,3 +19,23 @@ CREATE TABLE IF NOT EXISTS `source_openrec` (
 	KEY `update_time_idx` (`update_time`),
 	KEY `target_type_idx` (`target_type`)
  );
+
+
+ CREATE TABLE IF NOT EXISTS `source_twitch` (
+	uid bigint AUTO_INCREMENT COMMENT '自增UID',
+
+	`streamer_id` bigint DEFAULT NULL COMMENT 'streamer uid, 关联主播',
+	`url` text NOT NULL COMMENT '获取源数据地址',
+	`source` longtext DEFAULT NULL COMMENT '源数据',
+	`ext` json DEFAULT NULL COMMENT '扩展字段',
+	`serialize` blob DEFAULT NULL COMMENT '需要给下个任务传递 序列花数据, 非必要不用',
+	`update_time` Timestamp  NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新数据',
+	`error_msg` text DEFAULT NULL COMMENT '错误信息',
+
+	`target_type` varchar(64) NOT NULL COMMENT '目标类型',
+	`operator` int DEFAULT 0 COMMENT '操作标志位, 根据不同解析方法有不同标志',
+	PRIMARY KEY(`uid`),
+	KEY `operator_idx` (`operator`),
+	KEY `update_time_idx` (`update_time`),
+	KEY `target_type_idx` (`target_type`)
+ );

sql/remake_database.sh

@@ -0,0 +1,10 @@
+# /bin/bash
+USER=root
+HOST=127.0.0.1
+PORT=4000
+
+# mysql -h $HOST -u $USER -P $PORT -c "drop database intimate_source";
+# mysql -h $HOST -u $USER -P $PORT -c "drop database intimate_extractor";
+
+mysql -h $HOST -u $USER -P $PORT < ./intimate_extractor.sql;
+mysql -h $HOST -u $USER -P $PORT < ./intimate_source.sql;

store.go

@@ -1,8 +1,11 @@
 package intimate
 
 import (
+	"crypto/md5"
 	"database/sql"
+	"fmt"
 	"log"
+	"strings"
 	"time"
 
 	_ "github.com/go-sql-driver/mysql"
@@ -39,10 +42,20 @@ type IGetSet interface {
 type StoreSource struct {
 	table string
 	db    *sql.DB
+
+	popCount   int
 	errorCount int
 	errorLimit int
 }
 
+func (store *StoreSource) PopCount() int {
+	return store.popCount
+}
+
+func (store *StoreSource) Close() error {
+	return store.db.Close()
+}
+
 // NewSourceStore 创建一个存储实例
 func NewStoreSource(table string) *StoreSource {
 	db, err := sql.Open("mysql", InitConfig.Database.SourceURI)
@@ -69,7 +82,16 @@ func (store *StoreSource) errorAlarm(err error) {
 
 // Insert 插入数据
 func (store *StoreSource) Insert(isource IGet) {
-	_, err := store.db.Exec("insert into "+store.table+"(url, target_type, source,  ext, operator, error_msg, streamer_id) values(?,?,?,?,?,?,?)", isource.Get("Url"), isource.Get("TargetType"), isource.Get("Source"), isource.Get("Ext"), isource.Get("Operator"), isource.Get("ErrorMsg"), isource.Get("StreamerId"))
+	_, err := store.db.Exec("insert into "+store.table+"(url, target_type, source,  ext, operator, error_msg, streamer_id) values(?,?,?,?,?,?,?)", isource.Get("Url"), isource.Get("Target"), isource.Get("Source"), isource.Get("Ext"), isource.Get("Operator"), isource.Get("ErrorMsg"), isource.Get("StreamerId"))
+	if err != nil {
+		panic(err)
+	}
+}
+
+// Deduplicate 去重
+func (store *StoreSource) Deduplicate(target Target, field string) {
+	sql := `DELETE FROM ` + store.table + ` WHERE uid NOT IN (SELECT MAX(s.uid) FROM (SELECT uid, ` + field + ` FROM ` + store.table + ` force index(target_type_idx) WHERE target_type = "` + string(target) + `" ) s GROUP BY s.` + string(field) + `) ;`
+	_, err := store.db.Exec(sql)
 	if err != nil {
 		panic(err)
 	}
@@ -112,13 +134,13 @@ func (store *StoreSource) Restore(isource IGet) {
 }
 
 // Pop 弹出一条未处理的数据
-func (store *StoreSource) Pop(targetType string, operators ...int32) (*Source, error) {
+func (store *StoreSource) Pop(targetType Target, operators ...int32) (*Source, error) {
 
 	tx, err := store.db.Begin()
 	if err != nil {
 		return nil, err
 	}
-	var args = []interface{}{targetType}
+	var args = []interface{}{string(targetType)}
 	selectSQL := `select uid, url, target_type, source, ext, operator, update_time, streamer_id from ` + store.table + ` where target_type = ?`
 	if len(operators) == 0 {
 		selectSQL += " and operator = ?"
@@ -142,14 +164,16 @@ func (store *StoreSource) Pop(targetType string, operators ...int32) (*Source, e
 				log.Println(err)
 			}
 		}
+		store.popCount++
 	}()
 
 	s := &Source{}
 	// uid, url, target_type, source, ext, operator
-	err = row.Scan(&s.Uid, &s.Url, &s.TargetType, &s.Source, &s.Ext, &s.Operator, &s.UpdateTime, &s.StreamerId)
+	err = row.Scan(&s.Uid, &s.Url, &s.Target, &s.Source, &s.Ext, &s.Operator, &s.UpdateTime, &s.StreamerId)
 	if err != nil {
 		return nil, err
 	}
+
 	s.Set("LastOperator", s.Operator)
 	_, err = tx.Exec("update "+store.table+" set operator = ? where uid = ?", OperatorWait, s.Uid)
 	return s, nil
@@ -161,13 +185,25 @@ const StreamerTable string = "streamer"
 // CollectLogTable 采集日志表
 const CollectLogTable string = "collect_log"
 
+// StreamerListTable 主播表名称
+const StreamerListTable string = "streamer_list"
+
 type StoreExtractor struct {
 	db *sql.DB
 
+	popCount   int
 	errorCount int
 	errorLimit int
 }
 
+func (store *StoreExtractor) PopCount() int {
+	return store.popCount
+}
+
+func (store *StoreExtractor) Close() error {
+	return store.db.Close()
+}
+
 func (store *StoreExtractor) errorAlarm(err error) {
 	if err != nil {
 		log.Println("store error: ", err)
@@ -193,14 +229,14 @@ func NewStoreExtractor() *StoreExtractor {
 }
 
 // Pop 弹出一条未处理的数据
-func (store *StoreExtractor) Pop(platform string, operators ...int32) (*Streamer, error) {
+func (store *StoreExtractor) Pop(platform Platform, operators ...int32) (*Streamer, error) {
 
 	tx, err := store.db.Begin()
 	if err != nil {
 		return nil, err
 	}
-	var args = []interface{}{platform}
-	selectSQL := `select uid, update_time, user_id, update_url, is_update_streamer from ` + StreamerTable + ` where platform = ? and  TIMESTAMPDIFF(MINUTE , update_time, CURRENT_TIMESTAMP()) >= update_interval`
+	var args = []interface{}{string(platform)}
+	selectSQL := `select uid, update_time, user_id, update_url, is_update_streamer, update_interval from ` + StreamerTable + ` where platform = ? and  TIMESTAMPDIFF(MINUTE , update_time, CURRENT_TIMESTAMP()) >= update_interval`
 	if len(operators) == 0 {
 		selectSQL += " and operator = ?"
 		args = append(args, 0)
@@ -220,6 +256,7 @@ func (store *StoreExtractor) Pop(platform string, operators ...int32) (*Streamer
 				log.Println(err)
 			}
 		}
+		store.popCount++
 	}()
 
 	// log.Println(selectSQL + ` limit 1 for update`)
@@ -227,7 +264,7 @@ func (store *StoreExtractor) Pop(platform string, operators ...int32) (*Streamer
 
 	s := &Streamer{}
 	// uid, url, target_type, source, ext, operator
-	err = row.Scan(&s.Uid, &s.UpdateTime, &s.UserId, &s.UpdateUrl, &s.IsUpdateStreamer)
+	err = row.Scan(&s.Uid, &s.UpdateTime, &s.UserId, &s.UpdateUrl, &s.IsUpdateStreamer, &s.UpdateInterval)
 	if err != nil {
 		return nil, err
 	}
@@ -236,10 +273,52 @@ func (store *StoreExtractor) Pop(platform string, operators ...int32) (*Streamer
 	return s, nil
 }
 
+// UpdateStreamerList streamerlist表, 更新数据
+func (store *StoreExtractor) UpdateStreamerList(streamer IGet, fieldvalues ...interface{}) {
+	updateSQL := "UPDATE " + StreamerListTable + " SET "
+	var values []interface{}
+	for i := 0; i < len(fieldvalues); i += 2 {
+		field := fieldvalues[i]
+		values = append(values, fieldvalues[i+1])
+		updateSQL += field.(string) + " = ? "
+	}
+	updateSQL += "WHERE urlhash = ?"
+	values = append(values, streamer.Get("UrlHash"))
+	_, err := store.db.Exec(updateSQL, values...)
+	if err != nil {
+		panic(err)
+	}
+}
+
+// InsertStreamer streamerlist表, 插入数据
+func (store *StoreExtractor) InsertStreamerList(streamerlist IGet) (isExists bool) {
+	urlstr := streamerlist.Get("Url").(string)
+
+	_, err := store.db.Exec("insert into streamer_list(urlhash, url, platform, label, serialize, update_interval, error_msg, operator) values(?,?,?,?,?,?,?,?)",
+		fmt.Sprintf("%x", md5.Sum([]byte(urlstr))),
+		urlstr,
+		streamerlist.Get("Platform"),
+		streamerlist.Get("Label"),
+		streamerlist.Get("Serialize"),
+		streamerlist.Get("UpdateInterval"),
+		streamerlist.Get("ErrorMsg"),
+		streamerlist.Get("Operator"),
+	)
+
+	if err != nil {
+		if !strings.HasPrefix(err.Error(), "Error 1062") {
+			log.Println(err)
+		}
+		return true
+	}
+
+	return false
+}
+
 // InsertStreamer Streamer表, 插入数据
 func (store *StoreExtractor) InsertStreamer(streamer IGet) (isExists bool) {
 	// select uid from table where platform = ? and user_id = ?
-	selectSQL := "SELECT is_update_url FROM " + StreamerTable + " WHERE platform = ? AND user_id = ?"
+	selectSQL := "SELECT is_update_url, uid FROM " + StreamerTable + " WHERE platform = ? AND user_id = ?"
 	tx, err := store.db.Begin()
 	if err != nil {
 		panic(err)
@@ -258,14 +337,16 @@ func (store *StoreExtractor) InsertStreamer(streamer IGet) (isExists bool) {
 
 	row := tx.QueryRow(selectSQL+` LIMIT 1 FOR UPDATE`, streamer.Get("Platform"), streamer.Get("UserId"))
 	var isUpdateUrl bool
-	if err = row.Scan(&isUpdateUrl); err == nil {
+	var Uid int64
+	if err = row.Scan(&isUpdateUrl, &Uid); err == nil {
 		if isUpdateUrl {
 			tx.Exec("UPDATE "+StreamerTable+" SET update_url = ?", streamer.Get("UpdateUrl"))
 		}
+		streamer.(ISet).Set("Uid", Uid)
 		return true
 	}
 
-	_, err = tx.Exec("INSERT INTO "+StreamerTable+"(platform, user_id, update_url, update_time) VALUES(?,?,?,?);", streamer.Get("Platform"), streamer.Get("UserId"), streamer.Get("UpdateUrl"), time.Now().Add(-time.Minute*30))
+	_, err = tx.Exec("INSERT INTO "+StreamerTable+"(platform, user_id, update_url, tags, update_time) VALUES(?,?,?,?,?);", streamer.Get("Platform"), streamer.Get("UserId"), streamer.Get("UpdateUrl"), streamer.Get("Tags"), time.Now().Add(-time.Hour*100000))
 	if err != nil {
 		panic(err)
 	}
@@ -300,16 +381,36 @@ func (store *StoreExtractor) UpdateOperator(isource IGet) {
 }
 
 // UpdateStreamer Streamer表, 插入数据
-func (store *StoreExtractor) UpdateStreamer(isource IGet) {
-	_, err := store.db.Exec("UPDATE "+StreamerTable+" SET user_name = ?,  live_url = ?, channel = ?, latest_log_uid = ?, ext = ?, operator = ?, update_time = ? WHERE uid = ?;",
-		isource.Get("UserName"), isource.Get("LiveUrl"), isource.Get("Channel"), isource.Get("LatestLogUid"), isource.Get("Ext"), isource.Get("Operator"), isource.Get("UpdateTime"), isource.Get("Uid"))
+func (store *StoreExtractor) UpdateStreamer(streamer IGet) {
+	// log.Printf("UPDATE "+StreamerTable+" SET user_name = %v,  live_url = %v, channel = %v, latest_log_uid = %v, tags = %v, ext = %v, operator = %v, update_time = %v, update_interval = %v WHERE uid = %v", streamer.Get("UserName"), streamer.Get("LiveUrl"), streamer.Get("Channel"), streamer.Get("LatestLogUid"), streamer.Get("Tags"), streamer.Get("Ext"), streamer.Get("Operator"), streamer.Get("UpdateTime"), streamer.Get("UpdateInterval"), streamer.Get("Uid"))
+	_, err := store.db.Exec("UPDATE "+StreamerTable+" SET user_name = ?,  live_url = ?, channel = ?, latest_log_uid = ?, tags = ?, ext = ?, operator = ?, update_time = ?, update_interval = ? WHERE uid = ?",
+		streamer.Get("UserName"), streamer.Get("LiveUrl"), streamer.Get("Channel"), streamer.Get("LatestLogUid"), streamer.Get("Tags"), streamer.Get("Ext"), streamer.Get("Operator"), streamer.Get("UpdateTime"), streamer.Get("UpdateInterval"), streamer.Get("Uid"))
 	if err != nil {
 		panic(err)
 	}
 }
 
-// InsertCollectLog CollectLog表插入数据
-func (store *StoreExtractor) InsertCollectLog(isource IGet) int64 {
+// Update Streamer表, 更新指定的字段
+func (store *StoreExtractor) Update(streamer IGet, fieldvalues ...interface{}) {
+	updateSQL := "UPDATE " + StreamerTable + " SET "
+	var values []interface{}
+	for i := 0; i < len(fieldvalues); i += 2 {
+		field := fieldvalues[i]
+		values = append(values, fieldvalues[i+1])
+		updateSQL += field.(string) + " = ?,"
+	}
+	updateSQL = updateSQL[0 : len(updateSQL)-1]
+	updateSQL += "WHERE uid = ?"
+	values = append(values, streamer.Get("Uid"))
+	_, err := store.db.Exec(updateSQL, values...)
+	if err != nil {
+		log.Println(updateSQL)
+		panic(err)
+	}
+}
+
+// InsertClog CollectLog表插入数据
+func (store *StoreExtractor) InsertClog(clog IGet) int64 {
 	tx, err := store.db.Begin()
 
 	defer func() {
@@ -324,7 +425,7 @@ func (store *StoreExtractor) InsertCollectLog(isource IGet) int64 {
 	}
 
 	result, err := tx.Exec("insert  into "+CollectLogTable+"(streamer_uid, platform, user_id,  is_live_streaming, is_error, followers, views, giver, gratuity, live_title, live_start_time, live_end_time, update_time, tags, ext, error_msg) values(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
-		isource.Get("StreamerUid"), isource.Get("Platform"), isource.Get("UserId"), isource.Get("IsLiveStreaming"), isource.Get("IsError"), isource.Get("Followers"), isource.Get("Views"), isource.Get("Giver"), isource.Get("Gratuity"), isource.Get("LiveTitle"), isource.Get("LiveStartTime"), isource.Get("LiveEndTime"), isource.Get("UpdateTime"), isource.Get("Tags"), isource.Get("Ext"), isource.Get("ErrorMsg"),
+		clog.Get("StreamerUid"), clog.Get("Platform"), clog.Get("UserId"), clog.Get("IsLiveStreaming"), clog.Get("IsError"), clog.Get("Followers"), clog.Get("Views"), clog.Get("Giver"), clog.Get("Gratuity"), clog.Get("LiveTitle"), clog.Get("LiveStartTime"), clog.Get("LiveEndTime"), clog.Get("UpdateTime"), clog.Get("Tags"), clog.Get("Ext"), clog.Get("ErrorMsg"),
 	)
 	if err != nil {
 		panic(err)
@@ -335,7 +436,7 @@ func (store *StoreExtractor) InsertCollectLog(isource IGet) int64 {
 		panic(err)
 	}
 
-	_, err = tx.Exec("update "+StreamerTable+" set latest_log_uid = ? where uid = ?", logUid, isource.Get("StreamerUid"))
+	_, err = tx.Exec("update "+StreamerTable+" set latest_log_uid = ? where uid = ?", logUid, clog.Get("StreamerUid"))
 	if err = tx.Commit(); err != nil {
 		panic(err)
 	}

store_test.go

@@ -5,8 +5,7 @@ import (
 )
 
 func TestStoreInsert(t *testing.T) {
-	// ht := hunter.NewHunter(openrecRanking)
-	// ht.Execute()
+
 }
 
 func TestStoreInsertCase1(t *testing.T) {

supervisor_conf/createln.sh

@@ -0,0 +1 @@
+ln -sf `pwd`/*.conf /etc/supervisor/conf.d/

supervisor_conf/openrec_extractor.conf

@@ -0,0 +1,10 @@
+[supervisord]
+nodaemon=true
+
+[program:openrec_extractor]
+directory = /home/eson/test/intimate/bin/openrec_extractor/
+command= /home/eson/test/intimate/bin/openrec_extractor/openrec_extractor
+autorestart=true
+stderr_logfile=/home/eson/test/intimate/bin/openrec_extractor/log
+stderr_logfile_maxbytes=0
+stopsignal=QUIT

supervisor_conf/openrec_task1.conf

@@ -0,0 +1,10 @@
+[supervisord]
+nodaemon=true
+
+[program:openrec_ranking]
+directory = /home/eson/test/intimate/bin/openrec_task1
+command= /home/eson/test/intimate/bin/openrec_task1/openrec_task1
+autorestart=true
+stderr_logfile=/home/eson/test/intimate/bin/openrec_task1/log
+stderr_logfile_maxbytes=0
+stopsignal=QUIT

supervisor_conf/openrec_task2.conf

@@ -0,0 +1,12 @@
+[supervisord]
+nodaemon=true
+
+[program:openrec_source]
+directory = /home/eson/test/intimate/bin/openrec_task2/
+command= /home/eson/test/intimate/bin/openrec_task2/openrec_task2
+process_name=%(program_name)s_%(process_num)02d              ;多进程名称
+numprocs=4                                                   ;启动多个进程
+autorestart=true
+stderr_logfile=/home/eson/test/intimate/bin/openrec_task2/log
+stderr_logfile_maxbytes=0
+stopsignal=QUIT

supervisor_conf/test.conf

@@ -0,0 +1,12 @@
+# [supervisord]
+# nodaemon=true
+
+# [program:openrec_extractor]
+# directory = /home/eson/test/intimate/extractor/openrec_extractor
+# command= /home/eson/test/intimate/extractor/openrec_extractor/openrec_extractor 
+# autorestart=true
+# # stdout_logfile=/home/eson/test/intimate/extractor/openrec/stdout
+# # stdout_logfile_maxbytes=0
+# stderr_logfile=/home/eson/test/intimate/extractor/openrec/log
+# stderr_logfile_maxbytes=0
+# stopsignal=QUIT

supervisor_conf/twitcasting_extractor.conf

@@ -0,0 +1,14 @@
+[supervisord]
+nodaemon=false
+
+[program:twitcasting_extractor]
+environment=DISPLAY=":99"
+directory = /home/eson/test/intimate/bin/twitcasting_extractor/
+command= /home/eson/test/intimate/bin/twitcasting_extractor/twitcasting_extractor
+process_name=%(program_name)s_%(process_num)02d              ;多进程名称
+numprocs=3                                                   ;启动多个进程
+autorestart=true
+stderr_logfile=/home/eson/test/intimate/bin/twitcasting_extractor/log
+# stderr_logfile=%(supervisorctl.var.directory)s/log
+stderr_logfile_maxbytes=0
+stopsignal=QUIT

supervisor_conf/twitcasting_task1.conf

@@ -0,0 +1,13 @@
+[supervisord]
+nodaemon=false
+
+[program:twitcasting_task1]
+environment=DISPLAY=":99"
+directory = /home/eson/test/intimate/bin/twitcasting_task1/
+command= /home/eson/test/intimate/bin/twitcasting_task1/twitcasting_task1 
+# process_name=%(program_name)s_%(process_num)02d              ;多进程名称
+# numprocs=1                                                   ;启动多个进程
+autorestart=true
+stderr_logfile=/home/eson/test/intimate/bin/twitcasting_task1/log
+stderr_logfile_maxbytes=0
+stopsignal=QUIT

supervisor_conf/twitch_extractor.conf

@@ -0,0 +1,13 @@
+[supervisord]
+nodaemon=false
+
+[program:twitch_extractor]
+environment=DISPLAY=":99"
+directory = /home/eson/test/intimate/bin/twitch_extractor
+command= /home/eson/test/intimate/bin/twitch_extractor/twitch_extractor 
+process_name=%(program_name)s_%(process_num)02d              ;多进程名称
+numprocs=5                                                  ;启动多个进程
+autorestart=true
+stderr_logfile=/home/eson/test/intimate/bin/twitch_extractor/log
+stderr_logfile_maxbytes=0
+stopsignal=QUIT

supervisor_conf/twitch_extractor_p1.conf

@@ -0,0 +1,13 @@
+[supervisord]
+nodaemon=true
+
+[program:twitch_extractor_p1]
+environment=DISPLAY=":99",pac_proxy=http://localhost:1090/pac
+directory = /home/eson/test/intimate/bin/twitch_extractor
+command= /home/eson/test/intimate/bin/twitch_extractor/twitch_extractor 
+process_name=%(program_name)s_%(process_num)02d              ;多进程名称
+numprocs=2                                                  ;启动多个进程
+autorestart=true
+stderr_logfile=/home/eson/test/intimate/bin/twitch_extractor/log
+stderr_logfile_maxbytes=0
+stopsignal=QUIT

supervisor_conf/twitch_extractor_p2.conf

@@ -0,0 +1,13 @@
+[supervisord]
+nodaemon=false
+
+[program:twitch_extractor_p2]
+environment=DISPLAY=":99",pac_proxy=http://localhost:1090/pac1
+directory = /home/eson/test/intimate/bin/twitch_extractor
+command= /home/eson/test/intimate/bin/twitch_extractor/twitch_extractor 
+process_name=%(program_name)s_%(process_num)02d              ;多进程名称
+numprocs=2                                                  ;启动多个进程
+autorestart=true
+stderr_logfile=/home/eson/test/intimate/bin/twitch_extractor/log
+stderr_logfile_maxbytes=0
+stopsignal=QUIT

supervisor_conf/twitch_task2.conf

@@ -0,0 +1,13 @@
+[supervisord]
+nodaemon=false
+
+[program:twitch_task2]
+environment=DISPLAY=":99"
+directory = /home/eson/test/intimate/bin/twitch_task2
+command= /home/eson/test/intimate/bin/twitch_task2/twitch_task2 
+process_name=%(program_name)s_%(process_num)02d              ;多进程名称
+numprocs=6                                                   ;启动多个进程
+autorestart=true
+stderr_logfile=/home/eson/test/intimate/bin/twitch_task2/log
+stderr_logfile_maxbytes=0
+stopsignal=QUIT

supervisor_conf/xvfb.conf

@@ -0,0 +1,13 @@
+
+[supervisord]
+nodaemon=true
+
+[program:xvfb-99]
+# directory = /home/eson/test/intimate/extractor/openrec_extractor
+command=/usr/bin/Xvfb :99 -screen 0 1280x720x24 -ac -nolisten tcp -dpi 96 +extension RANDR -nolisten tcp
+autorestart=true
+# # stdout_logfile=/home/eson/test/intimate/extractor/openrec/stdout
+# # stdout_logfile_maxbytes=0
+# stderr_logfile=/home/eson/test/intimate/extractor/openrec/log
+# stderr_logfile_maxbytes=0
+# stopsignal=QUIT

table_list.go

@@ -6,5 +6,10 @@ type SourceTable string
 const (
 	// STOpenrec openrec源table名称
 	STOpenrec SourceTable = "source_openrec"
-)
 
+	// STTwitch twitch源table名称
+	STTwitch SourceTable = "source_twitch"
+
+	// STTwitcasting STTwitcasting源table名称
+	STTwitcasting SourceTable = "source_twitcasting"
+)

target_type_list.go

@@ -1,12 +1,18 @@
 package intimate
 
-// TargetType 源的 目标类型 列表
-type TargetType string
+// Target 源的 目标类型 列表
+type Target string
 
 const (
-	// TTOpenrecRanking openrec源TargetType名称
-	TTOpenrecRanking TargetType = "openrec_ranking"
+	// TOpenrecRanking 获取排名 Target名称
+	TOpenrecRanking Target = "openrec_ranking"
 
-	// TTOpenrecUser openrec源TargetType名称
-	TTOpenrecUser TargetType = "openrec_user"
+	// TOpenrecUser  获取用户列表 源Target名称
+	TOpenrecUser Target = "openrec_user"
+
+	// TTwitchChannel twitch 获取类别操作目标
+	TTwitchChannel Target = "twitch_channel"
+
+	// TTwitchUser twitch 获取类别操作目标
+	TTwitchUser Target = "twitch_user"
 )

tasks/openrec/openrec_task1/main.go

@@ -1,8 +1,5 @@
 package main
 
-import "github.com/474420502/hunter"
-
 func main() {
-	ht := hunter.NewHunter(openrecRanking)
-	ht.Execute()
+	Execute()
 }

tasks/openrec/openrec_task1/task_openrec.go

@@ -4,29 +4,24 @@ import (
 	"encoding/json"
 	"intimate"
 	"log"
-	"os"
-	"os/signal"
 	"strconv"
-	"sync/atomic"
-	"syscall"
 	"time"
 
-	"github.com/474420502/hunter"
+	"github.com/474420502/gcurl"
 	"github.com/tidwall/gjson"
 )
 
-var openrecRanking *OpenrecRanking
-
 // sstore  源存储实例, 为存储源数据的实现. 表格具体参考sql/intimate_source.sql
 var sstore *intimate.StoreSource = intimate.NewStoreSource(string(intimate.STOpenrec))
 
 // estore 解析存储连接实例
 var estore *intimate.StoreExtractor = intimate.NewStoreExtractor()
 
-func init() {
+// Execute 执行方法
+func Execute() {
 
-	openrecRanking = &OpenrecRanking{}
-	openrecRanking.PreCurlUrl = `curl 'https://public.openrec.tv/external/api/v5/channel-ranks?period=monthly&date=&tag=&page=1' \
+	ps := intimate.NewPerfectShutdown()
+	turl := `curl 'https://public.openrec.tv/external/api/v5/channel-ranks?period=monthly&date=&tag=&page=1' \
 	-H 'authority: public.openrec.tv' \
 	-H 'accept: application/json, text/javascript, */*; q=0.01' \
 	-H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36' \
@@ -39,43 +34,25 @@ func init() {
 	-H 'if-none-match: W/"25edb-aUYBdmLqZcr6DW4ZWKX9r2aqolg"' \
 	--compressed`
 
-}
+	g := gcurl.ParseRawCURL(turl)
+	tp := g.Temporary()
 
-// OpenrecRanking 获取排名任务
-type OpenrecRanking struct {
-	hunter.PreCurlUrl
-}
+	for !ps.IsClose() {
 
-// Execute 执行方法
-func (or *OpenrecRanking) Execute(cxt *hunter.TaskContext) {
-
-	var loop int32 = 1
-
-	go func() {
-		signalchan := make(chan os.Signal)
-		signal.Notify(signalchan, syscall.SIGKILL, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGSTOP)
-		log.Println("accept stop command:", <-signalchan)
-		atomic.StoreInt32(&loop, 0)
-	}()
-
-	for atomic.LoadInt32(&loop) > 0 {
-
-		resp, err := cxt.Hunt()
+		resp, err := tp.Execute()
 		if err != nil {
 			log.Println(err)
 			time.Sleep(time.Second * 2)
 			continue
 		}
 
-		tp := cxt.Temporary()
-
 		content := resp.Content()
-		if len(content) <= 200 { // 末页退出
+		if len(content) <= 200 { //末页时没有内容返回, 末页退出
 			finishpoint := time.Now()
-			log.Println("任务Ranking UserId结束休眠, 下次启动时间:", finishpoint.Add(time.Minute*60))
-			for time.Now().Sub(finishpoint) < time.Minute*60 {
+			log.Println("任务Ranking UserId结束休眠, 下次启动时间:", finishpoint.Add(time.Minute*120))
+			for time.Now().Sub(finishpoint) < time.Minute*120 {
 				time.Sleep(time.Second)
-				if atomic.LoadInt32(&loop) > 0 {
+				if ps.IsClose() {
 					return
 				}
 			}
@@ -93,16 +70,9 @@ func (or *OpenrecRanking) Execute(cxt *hunter.TaskContext) {
 			for _, User := range result.Array() {
 
 				userid := User.Get("channel.id").String()
-
-				// data := &intimate.Source{}
-				// data.Source = sql.NullString{String: userid, Valid: len(userid) > 0}
-				// data.Url = tp.GetRawURL()
-				// data.TargetType = string(intimate.TTOpenrecUser)
-				// sstore.Insert(data)
-
 				streamer := &intimate.Streamer{}
 				streamer.UserId = userid
-				streamer.Platform = string(intimate.Popenrec)
+				streamer.Platform = intimate.Popenrec
 
 				updateUrl := make(map[string]interface{})
 
@@ -122,17 +92,8 @@ func (or *OpenrecRanking) Execute(cxt *hunter.TaskContext) {
 			}
 		}
 
-		querys := tp.GetQuery()
-		page, err := strconv.Atoi(querys.Get("page"))
-		if err != nil {
-			log.Println(err)
-			return
-		}
-
-		page++
-		querys.Set("page", strconv.Itoa(page))
-		tp.SetQuery(querys)
-
+		// 修改url query 参数的page递增. 遍历所有页面
+		tp.QueryParam("page").IntAdd(1)
 		time.Sleep(time.Second * 1)
 	}
 }

tasks/openrec/openrec_task1/task_openrec_test.go

@@ -3,67 +3,14 @@ package main
 import (
 	"testing"
 	"time"
-
-	"github.com/tidwall/gjson"
-
-	"github.com/474420502/hunter"
 )
 
-// OpenrecRanking 获取排名任务
-type OpenrecRankingTest struct {
-	hunter.PreCurlUrl
-}
-
-// Execute 执行方法
-func (or *OpenrecRankingTest) Execute(cxt *hunter.TaskContext) {
-
-	resp, err := cxt.Hunt()
-	if err != nil {
-		panic(err)
-	}
-	t := cxt.GetShare("test").(*testing.T)
-	if !gjson.ValidBytes(resp.Content()) {
-		t.Error("source is not json format.")
-	}
-	result := gjson.GetBytes(resp.Content(), "0.rank")
-	if result.Int() != 1 {
-		t.Error("rank is error. result raw is ", result.Raw)
-	}
-
-	if cxt.Temporary().GetQuery().Get("page") != "1" {
-		t.Error("Temporary page error")
-	}
-	// t.Error(string(resp.Content()))
-}
-
-func TestRanking(t *testing.T) {
-	curlBash := `curl 'https://public.openrec.tv/external/api/v5/channel-ranks?period=monthly&date=&tag=&page=1' \
-	-H 'authority: public.openrec.tv' \
-	-H 'accept: application/json, text/javascript, */*; q=0.01' \
-	-H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36' \
-	-H 'origin: https://www.openrec.tv' \
-	-H 'sec-fetch-site: same-site' \
-	-H 'sec-fetch-mode: cors' \
-	-H 'sec-fetch-dest: empty' \
-	-H 'referer: https://www.openrec.tv/ranking' \
-	-H 'accept-language: zh-CN,zh;q=0.9' \
-	-H 'if-none-match: W/"25edb-aUYBdmLqZcr6DW4ZWKX9r2aqolg"' \
-	--compressed`
-
-	ort := &OpenrecRankingTest{}
-	ort.PreCurlUrl = hunter.PreCurlUrl(curlBash)
-	ht := hunter.NewHunter(ort)
-	ht.SetShare("test", t)
-	ht.Execute()
-}
-
 func TestTimeAdd(t *testing.T) {
 	finishpoint := time.Now()
 	time.Sleep(time.Second * 2)
 	t.Error(time.Now().Sub(finishpoint) > time.Second*1)
 }
 
-func TestRankingInsert(t *testing.T) {
-	ht := hunter.NewHunter(openrecRanking)
-	ht.Execute()
+func TestMain(t *testing.T) {
+	main()
 }

tasks/openrec/openrec_task2/main.go

@@ -1,8 +1,5 @@
 package main
 
-import "github.com/474420502/hunter"
-
 func main() {
-	ht := hunter.NewHunter(oer)
-	ht.Execute()
+	Execute()
 }

tasks/openrec/openrec_task2/task_openrec.go

@@ -5,21 +5,13 @@ import (
 	"encoding/json"
 	"intimate"
 	"log"
-	"os"
-	"os/signal"
-	"strconv"
-	"sync/atomic"
-	"syscall"
 	"time"
 
 	"github.com/474420502/gcurl"
+	"github.com/474420502/requests"
 	"github.com/tidwall/gjson"
-
-	"github.com/474420502/hunter"
 )
 
-var oer *OpenrecExtratorRanking
-
 // sstore  源存储实例, 为存储源数据的实现. 表格具体参考sql/intimate_source.sql
 var sstore *intimate.StoreSource = intimate.NewStoreSource(string(intimate.STOpenrec))
 
@@ -27,31 +19,20 @@ var sstore *intimate.StoreSource = intimate.NewStoreSource(string(intimate.STOpe
 var estore *intimate.StoreExtractor = intimate.NewStoreExtractor()
 
 func init() {
-	oer = &OpenrecExtratorRanking{}
-}
 
-// OpenrecExtratorRanking 获取用户信息
-type OpenrecExtratorRanking struct {
-	// Store *intimate.Store
 }
 
 // Execute 执行方法
-func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) {
+func Execute() {
 
-	var loop int32 = 1
-
-	go func() {
-		signalchan := make(chan os.Signal)
-		signal.Notify(signalchan, syscall.SIGKILL, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGSTOP)
-		log.Println("accept stop command:", <-signalchan)
-		atomic.StoreInt32(&loop, 0)
-	}()
+	ps := intimate.NewPerfectShutdown()
+	ses := requests.NewSession()
 
 	var lasterr error = nil
 
-	for atomic.LoadInt32(&loop) > 0 {
+	for !ps.IsClose() {
 
-		streamer, err := estore.Pop(string(intimate.Popenrec))
+		streamer, err := estore.Pop(intimate.Popenrec) //队列里弹出一个streamer行. 进行解析
 
 		if streamer == nil || err != nil {
 			if err != lasterr {
@@ -66,7 +47,7 @@ func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) {
 
 		var updateUrl map[string]string
 
-		err = json.Unmarshal(streamer.UpdateUrl.([]byte), &updateUrl)
+		err = json.Unmarshal(streamer.UpdateUrl.([]byte), &updateUrl) // 反序列化update_url, 里面存了需要采集的url
 		if err != nil {
 			log.Println(err)
 			continue
@@ -74,7 +55,8 @@ func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) {
 		// Check Userid
 
 		userUrl := updateUrl["user"]
-		tp := cxt.Session().Get(userUrl)
+		log.Println(userUrl)
+		tp := ses.Get(userUrl) // 获取user url页面数据
 		resp, err := tp.Execute()
 		streamer.UpdateTime = sql.NullTime{Time: time.Now(), Valid: true}
 
@@ -84,16 +66,16 @@ func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) {
 			continue
 		}
 
-		cookies := cxt.Session().GetCookies(tp.GetParsedURL())
+		cookies := ses.GetCookies(tp.GetParsedURL())
 
-		scurl := updateUrl["supporters"]
+		scurl := updateUrl["supporters"] //获取打赏者的数据
 		curl := gcurl.ParseRawCURL(scurl)
 		supportersSession := curl.CreateSession()
 
 		temporary := curl.CreateTemporary(supportersSession)
 		supportersSession.SetCookies(temporary.GetParsedURL(), cookies)
 		var supporters []string
-		for {
+		for { // supporters 数据需要登录信息. 下面为赋值 supporters链接获取的uid token random码
 
 			supportersQuery := temporary.GetQuery()
 
@@ -122,22 +104,23 @@ func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) {
 				log.Println(err)
 			}
 			supporterjson := gjson.ParseBytes(resp.Content())
-			supporterdata := supporterjson.Get("data")
+			supporterdata := supporterjson.Get("data") //解析supporters获取的json数据
 			if supporterdata.Type == gjson.Null {
 				break
 			}
 			supporters = append(supporters, string(resp.Content()))
 
-			page := supportersQuery.Get("page_number")
-			pageint, err := strconv.Atoi(page)
-			if err != nil {
-				log.Println(err)
-				break
-			}
-			pageint++
-			page = strconv.Itoa(pageint)
-			supportersQuery.Set("page_number", page)
-			temporary.SetQuery(supportersQuery)
+			temporary.QueryParam("page_number").IntAdd(1)
+			// page := supportersQuery.Get("page_number") // page_number 加1
+			// pageint, err := strconv.Atoi(page)
+			// if err != nil {
+			// 	log.Println(err)
+			// 	break
+			// }
+			// pageint++
+			// page = strconv.Itoa(pageint)
+			// supportersQuery.Set("page_number", page)
+			// temporary.SetQuery(supportersQuery)
 		}
 
 		// cookies := cxt.Session().GetCookies(wf.GetParsedURL())
@@ -147,7 +130,7 @@ func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) {
 		ext["html_user"] = string(resp.Content())
 
 		liveUrl := updateUrl["live"]
-		tp = cxt.Session().Get(liveUrl)
+		tp = ses.Get(liveUrl)
 		resp, err = tp.Execute()
 		if err != nil {
 			log.Println(err)
@@ -164,15 +147,18 @@ func (oer *OpenrecExtratorRanking) Execute(cxt *hunter.TaskContext) {
 			continue
 		}
 
-		streamer.Operator = int32(intimate.OperatorOK)
+		// streamer.Platform = intimate.Popenrec
+		streamer.UpdateInterval = 120
+		streamer.UpdateTime = sql.NullTime{Time: time.Now(), Valid: true}
+		streamer.Operator = 0
 
 		source := &intimate.Source{}
-		source.TargetType = string(intimate.TTOpenrecUser)
+		source.Target = intimate.TOpenrecUser
 		source.Ext = string(extJsonBytes)
 		source.StreamerId = sql.NullInt64{Int64: streamer.Uid, Valid: true}
 		sstore.Insert(source)
 
-		estore.UpdateOperator(streamer)
+		estore.UpdateStreamer(streamer)
 	}
 
 }

tasks/openrec/openrec_task2/task_openrec_test.go

@@ -2,11 +2,8 @@ package main
 
 import (
 	"testing"
-
-	"github.com/474420502/hunter"
 )
 
-func TestOpenrecUser(t *testing.T) {
-	ht := hunter.NewHunter(oer)
-	ht.Execute()
+func TestMain(t *testing.T) {
+	main()
 }

tasks/twitcasting/twitcasting_task1/.gitignore

@@ -0,0 +1,2 @@
+twitcasting_task1
+log

tasks/twitcasting/twitcasting_task1/error.html

@@ -0,0 +1,991 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN">
+<html>
+<head>
+    <title>Live with Tag: vocaloid  - TwitCasting</title>
+    <meta http-equiv="content-type" content="text/html; charset=UTF-8">
+<link rel="stylesheet" type="text/css" media="all" href="/css/mainstyle-us.css?1595400959">
+
+
+<meta name="viewport" content="width=device-width, user-scalable=no"/>
+
+
+<meta name="theme-color" content="#3381ff"/>
+<link href="/img/icon192.png" rel="icon" sizes="192x192"/>
+<link href="/img/icon72@2x.png" rel="apple-touch-icon" type="image/png"/>
+<link rel="stylesheet" href="/css/bootstrap.css?1596704154" type="text/css" media="all"/>
+<script type="text/javascript" src="/js/jquery-1.7.2.min.js"></script>
+<script crossorigin="anonymous" src="https://polyfill.io/v3/polyfill.min.js?features=default%2Ces2015%2Ces2016%2CIntersectionObserver%2CNodeList.prototype.forEach%2CNodeList.prototype.%40%40iterator%2CPromise.prototype.finally%2Cfetch"></script>
+<script type="text/javascript" src="/js/moiad.js?1526617899" defer></script>
+<script type="text/javascript" src="/js/Main.js?1597109494"></script>
+<meta name="format-detection" content="telephone=no">
+
+<script>(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
+(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
+m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
+})(window,document,'script','//www.google-analytics.com/analytics.js','ga');
+
+ga('create', 'UA-51336-17', 'auto');
+ga('require', 'displayfeatures');
+ga('send', 'pageview');   ga('create', 'UA-51336-19', 'auto', {'name': 'usTracker'});
+ga('usTracker.require', 'displayfeatures');
+ga('usTracker.send', 'pageview');</script>    <link rel="stylesheet" href="/css/search.css?1578279147" type="text/css" media="all"/>
+</head>
+<body  data-is-mobile="false">
+
+
+<div class="tw-global-header-locator">
+    <nav class="tw-global-header global-nav" data-lang="en"
+         data-from-device="false"
+         data-is-login="false"
+         data-is-mobile="false"
+             >
+        <div class="tw-global-header-container header-container">
+            <span class="tw-header-toggle-button">
+                <a class="tw-hamburger-icon" href="#">
+                    <span></span>
+                    <span></span>
+                    <span></span>
+                </a>
+            </span>
+            <div class="tw-global-header-primary">
+                                <h1 class="tw-global-header-logo">
+                    <a href="/">
+                        <div class="tw-global-header-logo-large">
+                            <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1077.3 202"><defs/><g id="レイヤー_2" data-name="レイヤー 2"><g id="Layer_1" data-name="Layer 1"><path d="M946.48 0l-6.46 30.27h15.89l-1.56 7.32h-25.86L936.46 0h10.02zm40.29 0l-7.98 37.59h-10.01L976.75 0h10.02zm23.91 0c1.72 18.37 2.42 27.2 2.48 28.92h.1c.76-1.72 5.12-10.66 14.6-28.92h10.72l-20.74 37.59h-12.39L1000.5 0zm66.62 0l-1.4 7.05h-15.4l-1.62 7.81h14.55l-1.46 7.06h-14.6l-1.77 8.46h15.83l-1.4 7.21h-25.64L1052.36 0h24.94zM106.37 27.88l-5.11 23.65-34.07-.37L44.1 159.99H12.68L36.15 51.16 0 51.53l5.1-23.65h101.27z" fill="#0472fc"/><path class="cls-1" d="M97.84 160L86.1 62.13h29.72c4.16 57.73 4.73 69.47 4.92 75h.37c2.09-5.67 6.44-16.65 33.51-75h34.82c2.65 56.4 3 69.65 3 75.15h.38c2.46-6.25 7-16.85 35.77-75.15h31.05L206.1 160h-35.58c-3.22-59.25-3.41-69.65-3.41-74.77h-.38c-1.51 4.55-4.92 13.63-33.31 74.77zM307.9 45.29h-29.72l5.49-24.42h29.53zM254.52 160l20.83-97.86h29.71L284.24 160zM375.08 31.66l-6.62 30.47h23.66l-4.17 21H364.1l-8.51 39.94a32.22 32.22 0 0 0-.95 7.38c0 6.06 3.41 9.65 10.79 9.65a31.49 31.49 0 0 0 9.84-1.7l-3.78 21.2a62.77 62.77 0 0 1-15.14 1.89c-20.83 0-31.24-10.22-31.24-27.26a74.81 74.81 0 0 1 1.9-14.76L335 83.14h-17.6l4.54-21h17.6L344 40.93zm136.45 25.17c-7.76-4-17.41-6.06-28-6.06-30.85 0-50.34 22.34-50.34 51.11 0 23.09 15.14 35.4 38.42 35.4a68.1 68.1 0 0 0 25.93-5.11l-4.17 25.54c-8.13 2.46-17 3.79-28.76 3.79-35.59 0-63.79-18.36-63.79-57.73 0-43.72 34.45-77.22 81.2-77.22 13.63 0 26.69 2.27 36.53 6.25zm58.66 58.3c-22.14 0-31 6.25-31 16.28 0 7.19 5.3 11.16 12.49 11.16 14.39 0 24.8-11.35 26.88-26.87a65.78 65.78 0 0 0-8.37-.57zM575.3 145c-7.95 12.3-19.3 16.84-33.12 16.84-18.74 0-31.61-11-31.61-28.39 0-26.31 26.88-34.83 56.78-34.83 3.6 0 8.71.19 13.63.57a24.1 24.1 0 0 0 .38-4.92c0-10-7-14.58-20.25-14.58-9.47 0-21.58 2.28-30.85 7L532.91 65a109.76 109.76 0 0 1 33.88-5.3c26.31 0 41.64 10.22 41.64 31.23 0 11-7.19 40.51-8.9 52a115.06 115.06 0 0 0-1.7 17H574a98.43 98.43 0 0 1 1.7-15zM693 86.17a52.92 52.92 0 0 0-24-5.87c-9.09 0-14.58 2.46-14.58 8.33 0 5.11 3.22 7.38 13.82 13.25 13.82 7.57 21.2 14.57 21.2 27.83 0 24.41-22.15 32.74-44.86 32.74-12.11 0-24.6-2.28-33.69-6.06l6.06-22.15c8 4 18.92 6.82 27.44 6.82 10.41 0 15-3.22 15-9.46 0-4.74-3.21-8.14-15.14-14.39-14.2-7.57-19.87-14.95-19.87-26.12 0-20.06 18.73-31.42 44.86-31.42a79.88 79.88 0 0 1 29.9 5.68zm67-54.51l-6.63 30.47H777l-4.16 21H749l-8.52 39.94a32.09 32.09 0 0 0-.94 7.38c0 6.06 3.4 9.65 10.79 9.65a31.48 31.48 0 0 0 9.83-1.7l-3.78 21.2a62.77 62.77 0 0 1-15.14 1.89c-20.82 0-31.23-10.22-31.23-27.26a75.49 75.49 0 0 1 1.89-14.76l8-36.34h-17.6l4.54-21h17.6l4.56-21.2zm70.78 13.63h-29.71l5.48-24.42h29.53zM777.4 160l20.82-97.86h29.72L807.12 160zm98.22-82.35c8.33-12.3 19.88-18 32-18C928.44 59.67 940 72.54 940 92a85.47 85.47 0 0 1-2.08 18.36L927.49 160h-29c3.22-15.52 6.43-31 9.84-46.57 1.33-6.43 2.27-12.3 2.27-16.08 0-8.9-4.35-14.58-13.44-14.58-15.14 0-23.09 13.63-29 40.89L860.48 160h-29.14l16.09-76.66c1.32-6.43 2.64-14 3.59-21.2h26.88c-.38 4.36-1.71 11-2.65 15.52zm146.87 24.42c0-12.49-6.63-21.39-18-21.39-14.77 0-27.07 16.84-27.07 36.53 0 12.11 5.49 21.2 17.41 21.2 15.92 0 27.66-17.6 27.66-36.34zm9.27-39.94H1060l-20.06 93.32c-6.43 30.09-23.28 46.55-59.62 46.55a114.39 114.39 0 0 1-38-7l6.62-22.9c10.6 4.92 21.77 7.76 31.8 7.76 20.06 0 30.09-9.84 35.2-36.91h-.37c-7.2 10.6-17.8 16.66-31.24 16.66-23.46 0-36.53-16.47-36.53-39.75 0-32.74 21.2-60.19 51.68-60.19 14.76 0 24.22 6.63 29 17.79h.37z" fill="#0472fc"/><path fill="#0472fc" d="M919.73 195H5.84l2.97-14h913.9l-2.98 14z" fill="#0472fc"/></g></g></svg>
+                        </div>
+                        <div class="tw-global-header-logo-small">
+                            <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><g data-name="レイヤー 2"><g data-name="Layer 1"><path fill="#fff" fill-rule="evenodd" d="M7.65 14.38a1.07 1.07 0 1 1 2.13 0 1.07 1.07 0 0 1-2.13 0z"/><path class="cls-1" fill="#fff" d="M17.79 21.89a11 11 0 0 1-1.33.67A11.43 11.43 0 0 1 3.9 20.1c-.14-.14-.28-.28-.41-.43q.31-.51.72-1.05a5.09 5.09 0 0 1 1-1A3 3 0 0 1 7.7 17c1 .23.68 1 .68 1 .87.34.32 1.38.32 1.38a.88.88 0 0 1 .05.15h.1a1.29 1.29 0 0 0 1.29-1.29v-1.33l1.73 1.16a.52.52 0 0 0 .51 0l1.8-1.21-2 .6h-.09l-1.93-.57v-.32l2 .58 2.29-.68-.06-.06-2-1.22a.53.53 0 0 0-.52 0l-1.72 1.07v-2a1.29 1.29 0 0 0-1.29-1.29H6.49a1.29 1.29 0 0 0-1.29 1.25v2.39A7.79 7.79 0 0 1 8.94 6l2.74-2.41c.15-.13.49-.43.74-.3s.11.4.09.45a4.27 4.27 0 0 0-.15 1.58 2.81 2.81 0 0 1 1.85-1c.43-.06.37.2.31.26-.67.78.77 1.41.77 1.41a7.78 7.78 0 0 1 .65 13.88 6.84 6.84 0 0 1 1.85 2.02z"/><path d="M14.91 13.63a.81.81 0 1 0 .81.81.81.81 0 0 0-.81-.81zM23.73 12a11.78 11.78 0 0 1-5.82 10.14A11.73 11.73 0 0 0 23.73 12zm-15 1.34a1.07 1.07 0 1 0 0 2.13 1.07 1.07 0 0 0 0-2.13z" fill="#0073ff" fill-rule="evenodd"/><path class="cls-3" fill="#0073ff" d="M23.73 12a11.78 11.78 0 0 1-5.82 10.14A11.73 11.73 0 0 0 23.73 12z"/><path class="cls-3" fill="#0073ff" d="M23.06 7.33A12 12 0 0 0 16.67.94a12.07 12.07 0 0 0-9.34 0A12 12 0 0 0 .94 7.33a12.07 12.07 0 0 0 0 9.34 12 12 0 0 0 6.39 6.39 12.07 12.07 0 0 0 9.34 0 12 12 0 0 0 6.39-6.39 12.07 12.07 0 0 0 0-9.34zm-6.6 15.23A11.43 11.43 0 0 1 3.9 20.1c-.14-.14-.28-.28-.41-.43q.31-.51.72-1.05a5.09 5.09 0 0 1 1-1A3 3 0 0 1 7.7 17c1 .23.68 1 .68 1 .87.34.32 1.38.32 1.38a.88.88 0 0 1 .05.15h.1a1.29 1.29 0 0 0 1.29-1.29v-1.33l1.73 1.16a.52.52 0 0 0 .51 0l1.8-1.21-2 .6h-.09l-1.93-.57v-.32l2 .58 2.29-.68-.06-.06-2-1.22a.53.53 0 0 0-.52 0l-1.72 1.07v-2a1.29 1.29 0 0 0-1.29-1.29H6.49a1.29 1.29 0 0 0-1.29 1.25v2.39A7.79 7.79 0 0 1 8.94 6l2.74-2.41c.15-.13.49-.43.74-.3s.11.4.09.45a4.27 4.27 0 0 0-.15 1.58 2.81 2.81 0 0 1 1.85-1c.43-.06.37.2.31.26-.67.78.77 1.41.77 1.41a7.78 7.78 0 0 1 .65 13.88 6.84 6.84 0 0 1 1.85 2.05 11 11 0 0 1-1.33.64zm-8.81-8.18a1.07 1.07 0 1 1 2.13 0 1.07 1.07 0 0 1-2.13 0zm10.27 7.75A11.78 11.78 0 0 0 23.73 12a11.73 11.73 0 0 1-5.81 10.13z"/><path class="cls-1" fill="#fff" d="M8.72 13.31a1.07 1.07 0 1 0 0 2.13 1.07 1.07 0 0 0 0-2.13zm0 1.62a.55.55 0 0 1-.55-.55.55.55 0 1 1 .55.55z"/></g></g></svg>                        </div>
+                    </a>
+                </h1>
+                                    <form class="tw-global-header-mobile-search-bar" id="global-nav-search" method="post"
+                          action="/search/text/">
+                        <input type="search" class="tw-global-header-mobile-search-input" id="search-query" name="q"
+                               value=""
+                               placeholder="Search">
+                        <button type="submit" class="tw-global-header-mobile-search-button" aria-label="Search">
+                            <svg viewBox="0 0 24 24" width="24" height="24" xmlns="http://www.w3.org/2000/svg"><path d="M17.36 15.837l6.326 6.109a1.029 1.029 0 1 1-1.43 1.48l-6.344-6.127a9.707 9.707 0 0 1-6.164 2.196C4.364 19.495 0 15.131 0 9.748A9.748 9.748 0 0 1 9.748 0c5.383 0 9.747 4.364 9.747 9.748a9.71 9.71 0 0 1-2.135 6.089zm-7.612 1.6a7.69 7.69 0 1 0 0-15.38 7.69 7.69 0 0 0 0 15.38z" fill="#8D9296"/></svg>
+                        </button>
+                    </form>
+                            </div>
+            <div class="tw-global-header-secondary">
+
+                                                            <div class="tw-global-header-nav-item tw-global-header-nav-item-broadcast" data-can-hover="true">
+                            <a class="tw-global-header-nav-item-default" href="/indexloginwindow.php?next=broadcaster">
+                            <span class="tw-global-header-icon-broadcast">
+                                <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" width="24" height="24"><path d="M22.286 17.031L18 15.108v-4.275-.942l4.287-1.922v9.062zm-6-2.864c0 2.046 0 3.4-.395 3.782s-1.785.387-3.889.387H6c-2.1 0-3.492 0-3.887-.387s-.4-1.736-.4-3.782v-3.334c0-2.046 0-3.4.4-3.782S3.9 6.664 6 6.664h6c2.1 0 3.494 0 3.89.387s.394 1.736.394 3.782zm1.654-6.084a3.248 3.248 0 0 0-.84-2.208C16.2 5 14.8 5 12 5H6c-2.8 0-4.2 0-5.1.872S0 8.112 0 10.833v3.334c0 2.72 0 4.089.9 4.96S3.2 20 6 20h6c2.8 0 4.2 0 5.1-.872a3.259 3.259 0 0 0 .84-2.211l6.06 2.72V5.363z" fill-rule="evenodd"/></svg>
+                            </span>
+                                <span>Broadcast</span>
+                            </a>
+                        </div>
+                        <div class="tw-global-header-nav-item tw-global-header-nav-item-notifier" data-can-hover="true" aria-selected="false">
+                            <a class="tw-global-header-nav-item-default"
+                               href="/tcairindex.php">Subscriptions</a>
+                        </div>
+                    
+                                            <div class="tw-global-header-nav-item tw-global-header-nav-item-login">
+                    <span class="tw-global-header-nav-item-highlight">
+                        Log In                        <span class="tw-global-header-icon-down"><svg viewBox="0 0 24 24" width="24" height="24" xmlns="http://www.w3.org/2000/svg"><path d="M.531 8.035c-.708-.708-.708-1.911 0-2.62l.39-.388c.707-.708 1.91-.708 2.619 0l8.46 8.46 8.46-8.46c.708-.708 1.912-.708 2.62 0l.389.389c.708.708.708 1.911 0 2.62L12 19.503.531 8.035z" fill="#000" fill-rule="evenodd"/></svg>
+</span>
+                    </span>
+                        </div>
+                        <div class="tw-global-header-login-popup-locator">
+                            <div class="tw-global-header-login-popup">
+                                <div class="tw-global-header-block">
+                                    <ul class="tw-global-header-login-button-list">
+                                        <li>
+                                            <a class="tw-global-header-login-button" href="#" data-social="twitter"
+                                               onclick="go_auth(); return false;">
+                                                <svg viewBox="0 0 24 24" width="24" height="24" xmlns="http://www.w3.org/2000/svg"><path d="M7.548 21.501c9.056 0 14.01-7.503 14.01-14.01 0-.213 0-.425-.015-.636A10.019 10.019 0 0 0 24 4.305a9.828 9.828 0 0 1-2.828.776 4.941 4.941 0 0 0 2.165-2.724 9.868 9.868 0 0 1-3.127 1.195 4.929 4.929 0 0 0-8.391 4.491A13.98 13.98 0 0 1 1.67 2.9a4.928 4.928 0 0 0 1.525 6.573A4.887 4.887 0 0 1 .96 8.855v.063a4.926 4.926 0 0 0 3.95 4.827 4.916 4.916 0 0 1-2.223.084 4.93 4.93 0 0 0 4.6 3.42A9.88 9.88 0 0 1 0 19.289a13.94 13.94 0 0 0 7.548 2.209" fill="#FFF" fill-rule="evenodd"/></svg>
+                                                Twitter                                            </a>
+                                        </li>
+                                        <li>
+                                            <a class="tw-global-header-login-button" href="#" data-social="facebook"
+                                               onclick="go_auth('f'); return false;">
+                                                <svg viewBox="0 0 24 24" width="24" height="24" xmlns="http://www.w3.org/2000/svg"><path d="M22.676 0H1.324C.593 0 0 .593 0 1.324v21.352C0 23.407.593 24 1.324 24h11.504v-9.281H9.703v-3.633h3.125V8.412c0-3.099 1.895-4.787 4.659-4.787.931-.002 1.862.045 2.789.14v3.24h-1.904c-1.506 0-1.8.712-1.8 1.763v2.313h3.6l-.467 3.633h-3.153V24h6.124c.731 0 1.324-.593 1.324-1.324V1.324C24 .593 23.407 0 22.676 0z" fill="#FFF" fill-rule="evenodd"/></svg>
+                                                Facebook                                            </a>
+                                        </li>
+                                        <li>
+                                            <a class="tw-global-header-login-button" href="#" data-social="cas"
+                                               onclick="go_auth('c'); return false;">
+                                                <img src="/img/icon192.png">
+                                                TwitCasting                                            </a>
+                                        </li>
+                                    </ul>
+                                    <span class="tw-global-header-terms">
+                                         By logging in, you are indicating that you have read and agreed to our <a href="/indexlicense.php">Terms of Service.</a>                                    </span>
+                                    <a class="tw-global-header-other-login" href="/indexloginwindow.php">Login with other SNS</a>
+                                </div>
+                                <div class="tw-global-header-block">
+                                    <span class="tw-global-header-block-title">Create New Account</span>
+                                    <a href="/indexcasregister.php" class="tw-button-secondary tw-button-large" data-is-block="true">Sign Up</a>
+                                    <form class="tw-global-header-login-form">
+                                        <input type="hidden" id="keep" checked>
+                                    </form>
+                                </div>
+                            </div>
+                        </div>
+                                                </div>
+
+            
+        </div>
+
+        
+        <div class="tw-sidebar">
+        <div class="tw-sidebar-content" data-tablet="false">
+            <section class="tw-sidebar-main-column">
+                <ul class="tw-sidebar-main-navi">
+                                        <li class="tw-sidebar-live" >
+                        <a href="/?ch0">
+                            <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 20 17"><path d="M19.7 5.6c-.1-1-1-1.8-2-1.9h-.2c-1.5-.1-3.1-.3-4.7-.4-1.6-.1-3.3-.1-5 0h-.4l1.2-1.5h.2c.6 0 1-.4 1-1 0-.5-.4-1-1-1-.5 0-1 .5-1 1V1L4.4 3.5c-.7 0-1.3.1-1.9.2-1.2.2-2 .9-2.1 2.1-.1.3-.2.7-.2 1.1v.7c-.1.8-.2 1.7-.2 2.6 0 1.3.1 2.6.3 4.1.1 1 .9 1.8 1.9 1.9 1 .1 1.9.2 2.7.3 1.3.1 2.6.2 3.6.2h.9c3.4 0 5.9-.2 8.3-.5 1.1-.1 1.9-.9 2-1.9.1-1 .2-2.3.3-3.7 0-1.7-.2-3.5-.3-5zm-1.5 5c-.1 1-.2 2.1-.3 3.6 0 .2-.2.3-.5.4-2.3.3-4.7.5-8 .5h-.9c-1 0-2.2-.1-3.5-.2-.8-.1-1.6-.1-2.6-.3-.1 0-.3-.2-.3-.4-.2-1.5-.3-2.7-.3-3.9 0-.9.1-1.7.1-2.6L2 7c0-.3.1-.7.1-1 .1-.4.2-.5.6-.5C4.3 5.2 6 5.1 7.9 5h4.8c1.5.1 3.1.3 4.6.4h.2c.2 0 .3.2.3.4.2 1.5.4 3.2.4 4.8z" fill-rule="evenodd" clip-rule="evenodd" fill="#9ca1a6"/></svg>                            Live                        </a>
+                    </li>
+                                        <li class="tw-sidebar-category tw-sidebar-category-category">
+                        <a href="#">
+                            <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 20 16"><path class="st0" d="M1 0h3c.3 0 .5.2.5.5v3c0 .3-.2.5-.5.5H1c-.3 0-.5-.2-.5-.5v-3C.5.2.7 0 1 0zm6 1h12c.3 0 .5.2.5.5v1c0 .3-.2.5-.5.5H7c-.3 0-.5-.2-.5-.5v-1c0-.3.2-.5.5-.5zM1 6h3c.3 0 .5.2.5.5v3c0 .3-.2.5-.5.5H1c-.3 0-.5-.2-.5-.5v-3c0-.3.2-.5.5-.5zm6 1h12c.3 0 .5.2.5.5v1c0 .3-.2.5-.5.5H7c-.3 0-.5-.2-.5-.5v-1c0-.3.2-.5.5-.5zm-6 5h3c.3 0 .5.2.5.5v3c0 .3-.2.5-.5.5H1c-.3 0-.5-.2-.5-.5v-3c0-.3.2-.5.5-.5zm6 1h12c.3 0 .5.2.5.5v1c0 .3-.2.5-.5.5H7c-.3 0-.5-.2-.5-.5v-1c0-.3.2-.5.5-.5z" fill="#9ca1a6"/></svg>                            Category                        </a>
+                    </li>
+                    <li class="tw-sidebar-ranking" >
+                        <a href="/rankingindex.php">
+                            <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 20 18"><path d="M17.8 5.4c-.4 0-.7-.3-.7-.7s.3-.7.7-.7.7.3.7.7-.3.7-.7.7zm-2.3 10.5c0 .1-.1.1-.1.1H4.6c-.1 0-.1-.1-.1-.1v-2.1h11v2.1zM1.5 4.6c0-.4.3-.7.7-.7s.7.3.7.7-.3.8-.7.8-.7-.4-.7-.8zm4.9 2.2c.4 0 .8-.1 1.1-.4l2-1.6c.1 0 .3.1.5.1h.3l2 1.6c.3.3.7.4 1.1.4.2 0 .4 0 .6-.1l2.1-.8.4.4-.8 5.7H4.4l-.8-5.7.3-.3 2 .7h.5zM10 2c.4 0 .7.3.7.7s-.3.7-.7.7-.7-.3-.7-.7.3-.7.7-.7zm7.8.5c-1.1 0-2.1.9-2.2 2l-2.2.7h-.1L11.8 4c.2-.4.4-.8.4-1.3 0-1.2-1-2.2-2.2-2.2s-2.2 1-2.2 2.2c0 .4.1.8.3 1.1L6.5 5.1h-.1l-2-.7c-.1-1.1-1-2-2.2-2C1 2.5 0 3.4 0 4.6c0 1.1.9 2.1 2 2.2l.8 6.2v2.9c0 1 .8 1.8 1.8 1.8h10.7c1 0 1.8-.8 1.8-1.8V13l.9-6.2c1.1-.1 2-1.1 2-2.2 0-1.2-1-2.1-2.2-2.1z" fill-rule="evenodd" clip-rule="evenodd" fill="#9ca1a6"/></svg>                            Ranking                        </a>
+                    </li>
+                                    </ul>
+            </section>
+                    </div>
+
+        <div class="tw-sidebar-category-list-wrapper">
+            <ul class="tw-sidebar-category-list">
+
+                                                    <li class="tw-sidebar-category-list-item">
+                        <span class="tw-sidebar-category-list-title">Category</span>
+                        <ul class="tw-sidebar-category-sub-list">
+                                                            <li class="tw-sidebar-category-sub-list-item">
+                                    <a href="/?genre=category_gaming">
+                                        Gaming                                        <span class="tw-sidebar-category-live-count">(100)</span>
+                                    </a>
+
+                                </li>
+                                                    </ul>
+                    </li>
+                            </ul>
+        </div>
+        <div class="tw-sidebar-background"></div>
+    </div>
+        </nav>
+</div>
+
+<div class="tw-global-header-spacer"></div>
+
+<script type="text/javascript" src="/js/GlobalHeader.js?1593738718" defer></script>
+
+
+<div id="mainwrapper" class="clearfix">
+
+    <div class="tw-page tw-basic-page clearfix">
+
+        <div id="showadsheader"
+     style="text-align:center;padding-top:8px;padding-bottom:8px;margin-top:8px;margin-bottom:-2px;overflow:hidden;max-height:90px;"><script type="text/javascript">
+    window.addEventListener("DOMContentLoaded", function () {
+        window.adLoader = Moiad.AdLoaderFactory.create(
+            window,
+            "https:\/\/gox.twitcasting.tv\/",
+            "https:\/\/gox.twitcasting.tv\/",
+            200
+        );
+    });
+</script><script type="text/javascript">
+    window.addEventListener("DOMContentLoaded", function () {
+        var tag = new Moiad.SingleAdTag();
+        tag.setElementId("showadsheader");
+        tag.setPlaceName("tcx_general_header_pc_responsive");
+        adLoader.activate(tag); 
+    });    
+</script></div>
+        <div class="tw-basic-page-left-column">
+
+            <h2>
+                Tag Search :
+                <a href="/search/tag/vocaloid" class="tag tag-large tag-info">vocaloid</a>             </h2>
+
+            <span class="tw-search-result-count">
+                <a href="#tw-search-result-live">Live(0)</a> /
+                <a href="#tw-search-result-user">User(28)</a>
+            </span>
+
+            
+            
+                            
+                
+                <div class="tw-search-result-live" id="tw-search-result-live">
+                                    </div>
+
+                                    <div class="tw-search-result-user" id="tw-search-result-user">
+                            <div class="tw-search-result-row">
+        <div class="tw-search-result-row-user">
+            <div class="clearfix">
+                    <div class="indent">
+        <div class="userimage32">
+                        <a href="/lhymecyas">
+                <img src="//s01.twitcasting.tv/img/twitter_bigger_6.png" width="32" height="32" class="icon32">
+            </a>
+                    </div>
+        <div class="usertext">
+
+                            <a href="/lhymecyas"><span class="username">月詠來夢@<wbr>きゃす</span></a>
+            
+            <div class="fullname">@LhymeCyas</div>        </div>
+    </div>
+                </div>
+            <div class="tw-search-result-row-user-desc">月詠來夢のきゃす垢</div>
+            <div class="tw-search-result-row-user-tags">
+                <a href="/search/tag/%E6%AD%8C%E3%81%A3%E3%81%A6%E3%81%BF%E3%81%9F" class="tag tag-mini tag-info">歌ってみた</a> <a href="/search/tag/vocaloid" class="tag tag-mini tag-info">vocaloid</a> <a href="/search/tag/%E9%BC%BB%E5%A3%B0" class="tag tag-mini tag-info">鼻声</a> <a href="/search/tag/%E5%88%9D%E8%A6%8B%E3%81%95%E3%82%93%E5%A4%A7%E6%AD%93%E8%BF%8E" class="tag tag-mini tag-info">初見さん大歓迎</a> <a href="/search/tag/%E9%9B%91%E8%AB%87" class="tag tag-mini tag-info">雑談</a>             </div>
+        </div>
+    </div>
+    <div class="tw-search-result-row">
+        <div class="tw-search-result-row-user">
+            <div class="clearfix">
+                    <div class="indent">
+        <div class="userimage32">
+                        <a href="/c:shiratama_dango">
+                <img src="//imagegw02.twitcasting.tv/image3s/img-twitcasting.s3-us-west-1.amazonaws.com/89/a3/54ac2466ad751_64.jpg" width="32" height="32" class="icon32">
+            </a>
+                    </div>
+        <div class="usertext">
+
+                            <a href="/c:shiratama_dango"><span class="username">SHIRATAMA</span></a>
+            
+            <div class="fullname">@c:<wbr>shiratama_<wbr>dango</div>        </div>
+    </div>
+                </div>
+            <div class="tw-search-result-row-user-desc">A creature which enjoy singing, humming while walking, and eating.</div>
+            <div class="tw-search-result-row-user-tags">
+                <a href="/search/tag/anisong" class="tag tag-mini tag-info">anisong</a> <a href="/search/tag/singing" class="tag tag-mini tag-info">singing</a> <a href="/search/tag/japanese" class="tag tag-mini tag-info">japanese</a> <a href="/search/tag/thai" class="tag tag-mini tag-info">thai</a> <a href="/search/tag/vocaloid" class="tag tag-mini tag-info">vocaloid</a>             </div>
+        </div>
+    </div>
+    <div class="tw-search-result-row">
+        <div class="tw-search-result-row-user">
+            <div class="clearfix">
+                    <div class="indent">
+        <div class="userimage32">
+                        <a href="/zunguri1459">
+                <img src="//s01.twitcasting.tv/img/twitter_bigger_6.png" width="32" height="32" class="icon32">
+            </a>
+                    </div>
+        <div class="usertext">
+
+                            <a href="/zunguri1459"><span class="username">ずんちゃ</span></a>
+            
+            <div class="fullname">@zunguri145<wbr>9</div>        </div>
+    </div>
+                </div>
+            <div class="tw-search-result-row-user-desc">はろはわゆ</div>
+            <div class="tw-search-result-row-user-tags">
+                <a href="/search/tag/%E5%88%9D%E9%9F%B3%E3%83%9F%E3%82%AF" class="tag tag-mini tag-info">初音ミク</a> <a href="/search/tag/%E3%83%9C%E3%82%AB%E3%83%AD" class="tag tag-mini tag-info">ボカロ</a> <a href="/search/tag/%E3%81%BF%E3%81%A3%E3%81%B1%E3%81%84" class="tag tag-mini tag-info">みっぱい</a> <a href="/search/tag/%E5%88%9D%E5%BF%83%E8%80%85" class="tag tag-mini tag-info">初心者</a> <a href="/search/tag/vocaloid" class="tag tag-mini tag-info">vocaloid</a>             </div>
+        </div>
+    </div>
+    <div class="tw-search-result-row">
+        <div class="tw-search-result-row-user">
+            <div class="clearfix">
+                    <div class="indent">
+        <div class="userimage32">
+                        <a href="/kyabet001">
+                <img src="//imagegw02.twitcasting.tv/image3s/pbs.twimg.com/profile_images/780186868507828224/8i-n-KG9_bigger.jpg" width="32" height="32" class="icon32">
+            </a>
+                    </div>
+        <div class="usertext">
+
+                            <a href="/kyabet001"><span class="username">したばま　<wbr>ましか</span></a>
+            
+            <div class="fullname">@kyabet001</div>        </div>
+    </div>
+                </div>
+            <div class="tw-search-result-row-user-desc">2Dホラーアクションシューティング「CARLA」を製作中です  PV: https://t.co/PXBDoaNFUV よかったらフォロ/フォロバオナシャス</div>
+            <div class="tw-search-result-row-user-tags">
+                <a href="/search/tag/anime" class="tag tag-mini tag-info">anime</a> <a href="/search/tag/vocaloid" class="tag tag-mini tag-info">vocaloid</a> <a href="/search/tag/game" class="tag tag-mini tag-info">game</a> <a href="/search/tag/%E8%BB%8A" class="tag tag-mini tag-info">車</a> <a href="/search/tag/5bp." class="tag tag-mini tag-info">5bp.</a>             </div>
+        </div>
+    </div>
+    <div class="tw-search-result-row">
+        <div class="tw-search-result-row-user">
+            <div class="clearfix">
+                    <div class="indent">
+        <div class="userimage32">
+                        <a href="/c:grog">
+                <img src="//imagegw02.twitcasting.tv/image3s/img-twitcasting.s3-us-west-1.amazonaws.com/24/32/56c86d014226d_64.jpg" width="32" height="32" class="icon32">
+            </a>
+                    </div>
+        <div class="usertext">
+
+                            <a href="/c:grog"><span class="username">ぐろぐ</span></a>
+            
+            <div class="fullname">@c:<wbr>grog</div>        </div>
+    </div>
+                </div>
+            <div class="tw-search-result-row-user-desc"></div>
+            <div class="tw-search-result-row-user-tags">
+                <a href="/search/tag/MUSIC" class="tag tag-mini tag-info">MUSIC</a> <a href="/search/tag/%E6%AD%8C" class="tag tag-mini tag-info">歌</a> <a href="/search/tag/utaite" class="tag tag-mini tag-info">utaite</a> <a href="/search/tag/vocaloid" class="tag tag-mini tag-info">vocaloid</a>             </div>
+        </div>
+    </div>
+    <div class="tw-search-result-row">
+        <div class="tw-search-result-row-user">
+            <div class="clearfix">
+                    <div class="indent">
+        <div class="userimage32">
+                        <a href="/nayu_notyou">
+                <img src="//imagegw02.twitcasting.tv/image3s/pbs.twimg.com/profile_images/860291445026037760/Ig-6IP48_bigger.jpg" width="32" height="32" class="icon32">
+            </a>
+                    </div>
+        <div class="usertext">
+
+                            <a href="/nayu_notyou"><span class="username">んぁゆ <wbr>(<wbr>NaYu)<wbr></span></a>
+            
+            <div class="fullname">@NaYu_<wbr>NotYou</div>        </div>
+    </div>
+                </div>
+            <div class="tw-search-result-row-user-desc">🇮🇩🛫🇦🇺 || Main Account || Korean Related: @NaYu_ha || Pull me into more hells- || icon: @AyaminTwT</div>
+            <div class="tw-search-result-row-user-tags">
+                <a href="/search/tag/%E6%AD%8C" class="tag tag-mini tag-info">歌</a> <a href="/search/tag/%E6%AD%8C%E3%81%86" class="tag tag-mini tag-info">歌う</a> <a href="/search/tag/%E3%83%9C%E3%82%AB%E3%83%AD" class="tag tag-mini tag-info">ボカロ</a> <a href="/search/tag/vocaloid" class="tag tag-mini tag-info">vocaloid</a> <a href="/search/tag/anisong" class="tag tag-mini tag-info">anisong</a>             </div>
+        </div>
+    </div>
+    <div class="tw-search-result-row">
+        <div class="tw-search-result-row-user">
+            <div class="clearfix">
+                    <div class="indent">
+        <div class="userimage32">
+                        <a href="/0konoha39">
+                <img src="//imagegw02.twitcasting.tv/image3s/pbs.twimg.com/profile_images/679256076361969664/-dQl81Pj_bigger.jpg" width="32" height="32" class="icon32">
+            </a>
+                    </div>
+        <div class="usertext">
+
+                            <a href="/0konoha39"><span class="username">このは@<wbr>モンハン勢</span></a>
+            
+            <div class="fullname">@0konoha39</div>        </div>
+    </div>
+                </div>
+            <div class="tw-search-result-row-user-desc">DIVA、白猫、モンハンが大好きな大学生です(((((└(:D」┌)┘)))))))</div>
+            <div class="tw-search-result-row-user-tags">
+                <a href="/search/tag/%E3%81%93%E3%81%AE%E3%81%AF" class="tag tag-mini tag-info">このは</a> <a href="/search/tag/DIVA" class="tag tag-mini tag-info">DIVA</a> <a href="/search/tag/vocaloid" class="tag tag-mini tag-info">vocaloid</a> <a href="/search/tag/%E5%88%9D%E9%9F%B3%E3%83%9F%E3%82%AF" class="tag tag-mini tag-info">初音ミク</a> <a href="/search/tag/%E5%88%9D%E5%BF%83%E8%80%85" class="tag tag-mini tag-info">初心者</a>             </div>
+        </div>
+    </div>
+    <div class="tw-search-result-row">
+        <div class="tw-search-result-row-user">
+            <div class="clearfix">
+                    <div class="indent">
+        <div class="userimage32">
+                        <a href="/f:100000242133373">
+                <img src="//imagegw02.twitcasting.tv/image3s/graph.facebook.com/100000242133373/picture?type=square" width="32" height="32" class="icon32">
+            </a>
+                    </div>
+        <div class="usertext">
+
+                            <a href="/f:100000242133373"><span class="username">f:<wbr>Egao <wbr>No <wbr>ShouJou</span></a>
+            
+            <div class="fullname">@f:<wbr>1000002421<wbr>33373</div>        </div>
+    </div>
+                </div>
+            <div class="tw-search-result-row-user-desc"></div>
+            <div class="tw-search-result-row-user-tags">
+                <a href="/search/tag/sumairu" class="tag tag-mini tag-info">sumairu</a> <a href="/search/tag/vocaloid" class="tag tag-mini tag-info">vocaloid</a> <a href="/search/tag/song" class="tag tag-mini tag-info">song</a> <a href="/search/tag/live" class="tag tag-mini tag-info">live</a>             </div>
+        </div>
+    </div>
+    <div class="tw-search-result-row">
+        <div class="tw-search-result-row-user">
+            <div class="clearfix">
+                    <div class="indent">
+        <div class="userimage32">
+                        <a href="/rubierin_sna">
+                <img src="//imagegw02.twitcasting.tv/image3s/pbs.twimg.com/profile_images/828473387009740801/_fhSN6z3_bigger.jpg" width="32" height="32" class="icon32">
+            </a>
+                    </div>
+        <div class="usertext">
+
+                            <a href="/rubierin_sna"><span class="username">みゆ</span></a>
+            
+            <div class="fullname">@rubierin_<wbr>sna</div>        </div>
+    </div>
+                </div>
+            <div class="tw-search-result-row-user-desc">sing! やってます✨</div>
+            <div class="tw-search-result-row-user-tags">
+                <a href="/search/tag/Thai" class="tag tag-mini tag-info">Thai</a> <a href="/search/tag/vocaloid" class="tag tag-mini tag-info">vocaloid</a> <a href="/search/tag/Anime" class="tag tag-mini tag-info">Anime</a> <a href="/search/tag/anisong" class="tag tag-mini tag-info">anisong</a> <a href="/search/tag/Smule" class="tag tag-mini tag-info">Smule</a>             </div>
+        </div>
+    </div>
+    <div class="tw-search-result-row">
+        <div class="tw-search-result-row-user">
+            <div class="clearfix">
+                    <div class="indent">
+        <div class="userimage32">
+                        <a href="/f:100004051574775">
+                <img src="//imagegw02.twitcasting.tv/image3s/graph.facebook.com/100004051574775/picture?type=square" width="32" height="32" class="icon32">
+            </a>
+                    </div>
+        <div class="usertext">
+
+                            <a href="/f:100004051574775"><span class="username">f:<wbr>Yuu&#039;ll <wbr>Be <wbr>Back</span></a>
+            
+            <div class="fullname">@f:<wbr>1000040515<wbr>74775</div>        </div>
+    </div>
+                </div>
+            <div class="tw-search-result-row-user-desc"></div>
+            <div class="tw-search-result-row-user-tags">
+                <a href="/search/tag/anime" class="tag tag-mini tag-info">anime</a> <a href="/search/tag/music" class="tag tag-mini tag-info">music</a> <a href="/search/tag/english" class="tag tag-mini tag-info">english</a> <a href="/search/tag/thai" class="tag tag-mini tag-info">thai</a> <a href="/search/tag/vocaloid" class="tag tag-mini tag-info">vocaloid</a>             </div>
+        </div>
+    </div>
+    <div class="tw-search-result-row">
+        <div class="tw-search-result-row-user">
+            <div class="clearfix">
+                    <div class="indent">
+        <div class="userimage32">
+                        <a href="/c:mongdang">
+                <img src="https://ssl.twitcasting.tv/img/twitcas_normal_0.png" width="32" height="32" class="icon32">
+            </a>
+                    </div>
+        <div class="usertext">
+
+                            <a href="/c:mongdang"><span class="username">Mongdang</span></a>
+            
+            <div class="fullname">@c:<wbr>Mongdang</div>        </div>
+    </div>
+                </div>
+            <div class="tw-search-result-row-user-desc"></div>
+            <div class="tw-search-result-row-user-tags">
+                <a href="/search/tag/Karaoke" class="tag tag-mini tag-info">Karaoke</a> <a href="/search/tag/Anisong" class="tag tag-mini tag-info">Anisong</a> <a href="/search/tag/vocaloid" class="tag tag-mini tag-info">vocaloid</a> <a href="/search/tag/Anime" class="tag tag-mini tag-info">Anime</a> <a href="/search/tag/%E6%AD%8C%E3%81%A3%E3%81%A6%E3%81%BF%E3%81%9F" class="tag tag-mini tag-info">歌ってみた</a>             </div>
+        </div>
+    </div>
+    <div class="tw-search-result-row">
+        <div class="tw-search-result-row-user">
+            <div class="clearfix">
+                    <div class="indent">
+        <div class="userimage32">
+                        <a href="/kamiy_ari">
+                <img src="//imagegw02.twitcasting.tv/image3s/pbs.twimg.com/profile_images/1235041741155078144/tIYD3zkT_bigger.jpg" width="32" height="32" class="icon32">
+            </a>
+                    </div>
+        <div class="usertext">
+
+                            <a href="/kamiy_ari"><span class="username">狼谷ありー</span></a>
+            
+            <div class="fullname">@kamiy_<wbr>ari</div>        </div>
+    </div>
+                </div>
+            <div class="tw-search-result-row-user-desc">歌が大好きです。</div>
+            <div class="tw-search-result-row-user-tags">
+                <a href="/search/tag/%E3%82%AD%E3%83%A3%E3%82%B9%E5%88%9D%E5%BF%83%E8%80%85%E3%81%A7%E3%81%99" class="tag tag-mini tag-info">キャス初心者です</a> <a href="/search/tag/%E3%83%92%E3%83%88%E3%82%AB%E3%83%A9" class="tag tag-mini tag-info">ヒトカラ</a> <a href="/search/tag/VOCALOID" class="tag tag-mini tag-info">VOCALOID</a>             </div>
+        </div>
+    </div>
+    <div class="tw-search-result-row">
+        <div class="tw-search-result-row-user">
+            <div class="clearfix">
+                    <div class="indent">
+        <div class="userimage32">
+                        <a href="/f:100000714321056">
+                <img src="//imagegw02.twitcasting.tv/image3s/graph.facebook.com/100000714321056/picture?type=square" width="32" height="32" class="icon32">
+            </a>
+                    </div>
+        <div class="usertext">
+
+                            <a href="/f:100000714321056"><span class="username">f:<wbr>Thamanan <wbr>Wittayacha<wbr>mnankul</span></a>
+            
+            <div class="fullname">@f:<wbr>1000007143<wbr>21056</div>        </div>
+    </div>
+                </div>
+            <div class="tw-search-result-row-user-desc">อ่าา ชอบเกรียนที่สุดเบยย อ๊ายยย ขอให้ได้เกรียน เกรียนในสิ่งที่ชอบ และไม่เดือดร้อนคนอื่น โอเคโน๊ะ อยากจะเกรียนเพลงใส่กันเชิญจ้า กร๊ากก ผมเกรียนได้ทุกคนอยู่แระ ..</div>
+            <div class="tw-search-result-row-user-tags">
+                <a href="/search/tag/Anime" class="tag tag-mini tag-info">Anime</a> <a href="/search/tag/singing" class="tag tag-mini tag-info">singing</a> <a href="/search/tag/vocaloid" class="tag tag-mini tag-info">vocaloid</a>             </div>
+        </div>
+    </div>
+    <div class="tw-search-result-row">
+        <div class="tw-search-result-row-user">
+            <div class="clearfix">
+                    <div class="indent">
+        <div class="userimage32">
+                        <a href="/hikamody">
+                <img src="//imagegw02.twitcasting.tv/image3s/pbs.twimg.com/profile_images/976466557734813696/8jowWEM1_bigger.jpg" width="32" height="32" class="icon32">
+            </a>
+                    </div>
+        <div class="usertext">
+
+                            <a href="/hikamody"><span class="username">hika</span></a>
+            
+            <div class="fullname">@hikamody</div>        </div>
+    </div>
+                </div>
+            <div class="tw-search-result-row-user-desc">성인/잡덕/Bi🌈/Feminist/Flexitarian</div>
+            <div class="tw-search-result-row-user-tags">
+                <a href="/search/tag/%E6%AD%8C" class="tag tag-mini tag-info">歌</a> <a href="/search/tag/vocaloid" class="tag tag-mini tag-info">vocaloid</a> <a href="/search/tag/j-pop" class="tag tag-mini tag-info">j-pop</a>             </div>
+        </div>
+    </div>
+    <div class="tw-search-result-row">
+        <div class="tw-search-result-row-user">
+            <div class="clearfix">
+                    <div class="indent">
+        <div class="userimage32">
+                        <a href="/katie_diva_xoxo">
+                <img src="//imagegw02.twitcasting.tv/image3s/pbs.twimg.com/profile_images/649306799061884928/yBwmfjuO_normal.png" width="32" height="32" class="icon32">
+            </a>
+                    </div>
+        <div class="usertext">
+
+                            <a href="/katie_diva_xoxo"><span class="username">totally <wbr>katië <wbr>123</span></a>
+            
+            <div class="fullname">@katie_<wbr>diva_<wbr>xoxo</div>        </div>
+    </div>
+                </div>
+            <div class="tw-search-result-row-user-desc">❤️❤️❤️zoella</div>
+            <div class="tw-search-result-row-user-tags">
+                <a href="/search/tag/Music" class="tag tag-mini tag-info">Music</a> <a href="/search/tag/singing" class="tag tag-mini tag-info">singing</a> <a href="/search/tag/vocaloid" class="tag tag-mini tag-info">vocaloid</a>             </div>
+        </div>
+    </div>
+    <div class="tw-search-result-row">
+        <div class="tw-search-result-row-user">
+            <div class="clearfix">
+                    <div class="indent">
+        <div class="userimage32">
+                        <a href="/c:vlemvpe">
+                <img src="//imagegw02.twitcasting.tv/image3s/img-twitcasting.s3-us-west-1.amazonaws.com/26/7f/54c62b8de296a_64.jpg" width="32" height="32" class="icon32">
+            </a>
+                    </div>
+        <div class="usertext">
+
+                            <a href="/c:vlemvpe"><span class="username">Fujisaki <wbr>Hitomi</span></a>
+            
+            <div class="fullname">@c:<wbr>vlemvpe</div>        </div>
+    </div>
+                </div>
+            <div class="tw-search-result-row-user-desc">Fujisaki Hitomi
+https://www.facebook.com/ciazfah2
+คนไทยค่า :D</div>
+            <div class="tw-search-result-row-user-tags">
+                <a href="/search/tag/thai" class="tag tag-mini tag-info">thai</a> <a href="/search/tag/singing" class="tag tag-mini tag-info">singing</a> <a href="/search/tag/vocaloid" class="tag tag-mini tag-info">vocaloid</a> <a href="/search/tag/anisong" class="tag tag-mini tag-info">anisong</a>             </div>
+        </div>
+    </div>
+    <div class="tw-search-result-row">
+        <div class="tw-search-result-row-user">
+            <div class="clearfix">
+                    <div class="indent">
+        <div class="userimage32">
+                        <a href="/c:hatune3625">
+                <img src="//imagegw02.twitcasting.tv/image3s/img-twitcasting.s3-us-west-1.amazonaws.com/9a/a5/57d36c0a68ee1_64.jpg" width="32" height="32" class="icon32">
+            </a>
+                    </div>
+        <div class="usertext">
+
+                            <a href="/c:hatune3625"><span class="username">初音@<wbr>yu卍nan</span></a>
+            
+            <div class="fullname">@c:<wbr>hatune3625<wbr></div>        </div>
+    </div>
+                </div>
+            <div class="tw-search-result-row-user-desc">初音ミク大好き</div>
+            <div class="tw-search-result-row-user-tags">
+                <a href="/search/tag/%E5%88%9D%E9%9F%B3%E3%83%9F%E3%82%AF" class="tag tag-mini tag-info">初音ミク</a> <a href="/search/tag/%E9%9F%B3%E3%82%B2%E3%83%BC" class="tag tag-mini tag-info">音ゲー</a> <a href="/search/tag/vocaloid" class="tag tag-mini tag-info">vocaloid</a> <a href="/search/tag/%E3%82%B3%E3%83%A1%E3%83%B3%E3%83%88%E6%B0%97%E8%BB%BD%E3%81%AB" class="tag tag-mini tag-info">コメント気軽に</a>             </div>
+        </div>
+    </div>
+    <div class="tw-search-result-row">
+        <div class="tw-search-result-row-user">
+            <div class="clearfix">
+                    <div class="indent">
+        <div class="userimage32">
+                        <a href="/c:sara21ren">
+                <img src="https://ssl.twitcasting.tv/img/twitcas_normal_6.png" width="32" height="32" class="icon32">
+            </a>
+                    </div>
+        <div class="usertext">
+
+                            <a href="/c:sara21ren"><span class="username">Sarah</span></a>
+            
+            <div class="fullname">@c:<wbr>sara21ren</div>        </div>
+    </div>
+                </div>
+            <div class="tw-search-result-row-user-desc">Hello. It&#039;s Sarah (: I play the piano and the guitar. Sometimes sing. Ttm hmu!!</div>
+            <div class="tw-search-result-row-user-tags">
+                <a href="/search/tag/music" class="tag tag-mini tag-info">music</a> <a href="/search/tag/%E6%AD%8C" class="tag tag-mini tag-info">歌</a> <a href="/search/tag/Teen" class="tag tag-mini tag-info">Teen</a> <a href="/search/tag/english" class="tag tag-mini tag-info">english</a> <a href="/search/tag/vocaloid" class="tag tag-mini tag-info">vocaloid</a>             </div>
+        </div>
+    </div>
+    <div class="tw-search-result-row">
+        <div class="tw-search-result-row-user">
+            <div class="clearfix">
+                    <div class="indent">
+        <div class="userimage32">
+                        <a href="/kawazu1816">
+                <img src="//imagegw02.twitcasting.tv/image3s/pbs.twimg.com/profile_images/519776816798498817/2mQH2_CT_normal.jpeg" width="32" height="32" class="icon32">
+            </a>
+                    </div>
+        <div class="usertext">
+
+                            <a href="/kawazu1816"><span class="username">カワズ先輩≠カエル先輩</span></a>
+            
+            <div class="fullname">@kawazu1816<wbr></div>        </div>
+    </div>
+                </div>
+            <div class="tw-search-result-row-user-desc">愛したって、愛されたっていいじゃないか</div>
+            <div class="tw-search-result-row-user-tags">
+                <a href="/search/tag/%E5%88%9D%E5%BF%83%E8%80%85" class="tag tag-mini tag-info">初心者</a> <a href="/search/tag/%E5%88%9D%E8%A6%8B%E3%81%95%E3%82%93%E5%A4%A7%E6%AD%93%E8%BF%8E%21" class="tag tag-mini tag-info">初見さん大歓迎!</a> <a href="/search/tag/%E3%82%AB%E3%83%A9%E3%82%AA%E3%82%B1" class="tag tag-mini tag-info">カラオケ</a> <a href="/search/tag/%E7%8C%AB" class="tag tag-mini tag-info">猫</a> <a href="/search/tag/vocaloid" class="tag tag-mini tag-info">vocaloid</a>             </div>
+        </div>
+    </div>
+    <div class="tw-search-result-row">
+        <div class="tw-search-result-row-user">
+            <div class="clearfix">
+                    <div class="indent">
+        <div class="userimage32">
+                        <a href="/varenturu">
+                <img src="//imagegw02.twitcasting.tv/image3s/pbs.twimg.com/profile_images/846293033062563840/Hfw-YvYT_bigger.jpg" width="32" height="32" class="icon32">
+            </a>
+                    </div>
+        <div class="usertext">
+
+                            <a href="/varenturu"><span class="username">ヴァレン（<wbr>実況者）<wbr></span></a>
+            
+            <div class="fullname">@varenturu</div>        </div>
+    </div>
+                </div>
+            <div class="tw-search-result-row-user-desc">不定期で実況動画あげていく予定です。 よろしくお願いします！！ 異常なほどな誤字脱字etc...沢山のハプニングがあると思います。 温かい目で見てやってくれればありが..</div>
+            <div class="tw-search-result-row-user-tags">
+                <a href="/search/tag/%E6%AD%8C%E3%81%A3%E3%81%A6%E3%81%BF%E3%81%9F" class="tag tag-mini tag-info">歌ってみた</a> <a href="/search/tag/VOCALOID" class="tag tag-mini tag-info">VOCALOID</a> <a href="/search/tag/Fate" class="tag tag-mini tag-info">Fate</a> <a href="/search/tag/%E6%9D%B1%E6%96%B9" class="tag tag-mini tag-info">東方</a> <a href="/search/tag/%E3%82%A2%E3%83%8B%E3%82%BD%E3%83%B3" class="tag tag-mini tag-info">アニソン</a>             </div>
+        </div>
+    </div>
+    <div class="tw-search-result-row">
+        <div class="tw-search-result-row-user">
+            <div class="clearfix">
+                    <div class="indent">
+        <div class="userimage32">
+                        <a href="/c:iasoa020">
+                <img src="//imagegw02.twitcasting.tv/image3s/img-twitcasting.s3-us-west-1.amazonaws.com/75/c8/5f257e6bce867_64.jpg" width="32" height="32" class="icon32">
+            </a>
+                    </div>
+        <div class="usertext">
+
+                            <a href="/c:iasoa020"><span class="username">BARI</span></a>
+            
+            <div class="fullname">@c:<wbr>iasoa020</div>        </div>
+    </div>
+                </div>
+            <div class="tw-search-result-row-user-desc">歌を歌います。少しでもプラスに慣れればと思います。</div>
+            <div class="tw-search-result-row-user-tags">
+                <a href="/search/tag/%E3%83%9F%E3%83%A5%E3%83%BC%E3%82%B8%E3%82%AB%E3%83%AB" class="tag tag-mini tag-info">ミュージカル</a> <a href="/search/tag/%E5%A3%B0%E6%A5%BD" class="tag tag-mini tag-info">声楽</a> <a href="/search/tag/%E6%AD%8C" class="tag tag-mini tag-info">歌</a> <a href="/search/tag/VOCALOID" class="tag tag-mini tag-info">VOCALOID</a>             </div>
+        </div>
+    </div>
+    <div class="tw-search-result-row">
+        <div class="tw-search-result-row-user">
+            <div class="clearfix">
+                    <div class="indent">
+        <div class="userimage32">
+                        <a href="/f:100007528892324">
+                <img src="//imagegw02.twitcasting.tv/image3s/graph.facebook.com/100007528892324/picture?type=square" width="32" height="32" class="icon32">
+            </a>
+                    </div>
+        <div class="usertext">
+
+                            <a href="/f:100007528892324"><span class="username">f:<wbr>Xio <wbr>Steph</span></a>
+            
+            <div class="fullname">@f:<wbr>1000075288<wbr>92324</div>        </div>
+    </div>
+                </div>
+            <div class="tw-search-result-row-user-desc"></div>
+            <div class="tw-search-result-row-user-tags">
+                <a href="/search/tag/anime" class="tag tag-mini tag-info">anime</a> <a href="/search/tag/music" class="tag tag-mini tag-info">music</a> <a href="/search/tag/radio" class="tag tag-mini tag-info">radio</a> <a href="/search/tag/vocaloid" class="tag tag-mini tag-info">vocaloid</a> <a href="/search/tag/japan" class="tag tag-mini tag-info">japan</a>             </div>
+        </div>
+    </div>
+    <div class="tw-search-result-row">
+        <div class="tw-search-result-row-user">
+            <div class="clearfix">
+                    <div class="indent">
+        <div class="userimage32">
+                        <a href="/f:489672901190991">
+                <img src="//imagegw02.twitcasting.tv/image3s/graph.facebook.com/489672901190991/picture?type=square" width="32" height="32" class="icon32">
+            </a>
+                    </div>
+        <div class="usertext">
+
+                            <a href="/f:489672901190991"><span class="username">f:<wbr>Kama <wbr>Jewell <wbr>Greco</span></a>
+            
+            <div class="fullname">@f:<wbr>4896729011<wbr>90991</div>        </div>
+    </div>
+                </div>
+            <div class="tw-search-result-row-user-desc"></div>
+            <div class="tw-search-result-row-user-tags">
+                <a href="/search/tag/Singing" class="tag tag-mini tag-info">Singing</a> <a href="/search/tag/acoustic" class="tag tag-mini tag-info">acoustic</a> <a href="/search/tag/vocaloid" class="tag tag-mini tag-info">vocaloid</a> <a href="/search/tag/singerwongwriter" class="tag tag-mini tag-info">singerwongwriter</a>             </div>
+        </div>
+    </div>
+    <div class="tw-search-result-row">
+        <div class="tw-search-result-row-user">
+            <div class="clearfix">
+                    <div class="indent">
+        <div class="userimage32">
+                        <a href="/kiryu_hanabusa">
+                <img src="//imagegw02.twitcasting.tv/image3s/pbs.twimg.com/profile_images/1189080132373204992/YF4bSxAB_bigger.jpg" width="32" height="32" class="icon32">
+            </a>
+                    </div>
+        <div class="usertext">
+
+                            <a href="/kiryu_hanabusa"><span class="username">英 <wbr>姫柳/<wbr>かんじむずかしい</span></a>
+            
+            <div class="fullname">@Kiryu_<wbr>Hanabusa</div>        </div>
+    </div>
+                </div>
+            <div class="tw-search-result-row-user-desc">YouTuberになるために準備中です。 基本受けの体制なので話し掛けてもらうまでROMってることが多いです。気軽にお声掛けください！ codevein/競馬/麻雀/雀魂/FPS #姫柳の..</div>
+            <div class="tw-search-result-row-user-tags">
+                <a href="/search/tag/BUMPOFCHICKEN" class="tag tag-mini tag-info">BUMPOFCHICKEN</a> <a href="/search/tag/%E3%82%A2%E3%83%8B%E3%83%A1%E3%82%BD%E3%83%B3%E3%82%B0" class="tag tag-mini tag-info">アニメソング</a> <a href="/search/tag/VOCALOID" class="tag tag-mini tag-info">VOCALOID</a> <a href="/search/tag/%E3%82%B3%E3%83%A1%E3%83%B3%E3%83%88%E6%AD%93%E8%BF%8E" class="tag tag-mini tag-info">コメント歓迎</a> <a href="/search/tag/%E5%88%9D%E8%A6%8B%E3%81%95%E3%82%93%E5%A4%A7%E6%AD%93%E8%BF%8E" class="tag tag-mini tag-info">初見さん大歓迎</a>             </div>
+        </div>
+    </div>
+    <div class="tw-search-result-row">
+        <div class="tw-search-result-row-user">
+            <div class="clearfix">
+                    <div class="indent">
+        <div class="userimage32">
+                        <a href="/mujamiku">
+                <img src="//imagegw02.twitcasting.tv/image3s/pbs.twimg.com/profile_images/972119417747161089/XiTi7dBX_bigger.jpg" width="32" height="32" class="icon32">
+            </a>
+                    </div>
+        <div class="usertext">
+
+                            <a href="/mujamiku"><span class="username">ムジャオ＠<wbr>3/<wbr>9アイコン変わりました</span></a>
+            
+            <div class="fullname">@mujamiku</div>        </div>
+    </div>
+                </div>
+            <div class="tw-search-result-row-user-desc">ただのザコなボカロファンです。</div>
+            <div class="tw-search-result-row-user-tags">
+                <a href="/search/tag/%E5%88%9D%E9%9F%B3%E3%83%9F%E3%82%AF" class="tag tag-mini tag-info">初音ミク</a> <a href="/search/tag/%E3%83%9F%E3%82%AF%E3%82%B3%E3%83%AC" class="tag tag-mini tag-info">ミクコレ</a> <a href="/search/tag/vocaloid" class="tag tag-mini tag-info">vocaloid</a> <a href="/search/tag/%E9%9B%91%E8%AB%87" class="tag tag-mini tag-info">雑談</a> <a href="/search/tag/%E5%88%9D%E8%A6%8B%E3%81%95%E3%82%93%E5%A4%A7%E6%AD%93%E8%BF%8E" class="tag tag-mini tag-info">初見さん大歓迎</a>             </div>
+        </div>
+    </div>
+    <div class="tw-search-result-row">
+        <div class="tw-search-result-row-user">
+            <div class="clearfix">
+                    <div class="indent">
+        <div class="userimage32">
+                        <a href="/porkyuupine">
+                <img src="//imagegw02.twitcasting.tv/image3s/pbs.twimg.com/profile_images/838292064991236096/dx8lqDcW_bigger.jpg" width="32" height="32" class="icon32">
+            </a>
+                    </div>
+        <div class="usertext">
+
+                            <a href="/porkyuupine"><span class="username">(<wbr>๑•̀ω•́๑)<wbr></span></a>
+            
+            <div class="fullname">@porkyuupin<wbr>e</div>        </div>
+    </div>
+                </div>
+            <div class="tw-search-result-row-user-desc">❤ VOCALOID　｜　歌い手　｜　アニメと漫画 ❤ シンガポールからのファンです、よろしく〜☆ 　</div>
+            <div class="tw-search-result-row-user-tags">
+                <a href="/search/tag/bored" class="tag tag-mini tag-info">bored</a> <a href="/search/tag/music" class="tag tag-mini tag-info">music</a> <a href="/search/tag/%E6%AD%8C" class="tag tag-mini tag-info">歌</a> <a href="/search/tag/vocaloid" class="tag tag-mini tag-info">vocaloid</a> <a href="/search/tag/%E3%83%9C%E3%82%AB%E3%83%AD" class="tag tag-mini tag-info">ボカロ</a>             </div>
+        </div>
+    </div>
+    <div class="tw-search-result-row">
+        <div class="tw-search-result-row-user">
+            <div class="clearfix">
+                    <div class="indent">
+        <div class="userimage32">
+                        <a href="/c:mimingdayo">
+                <img src="//imagegw02.twitcasting.tv/image3s/img-twitcasting.s3-us-west-1.amazonaws.com/29/14/586d949bef0e6_64.jpg" width="32" height="32" class="icon32">
+            </a>
+                    </div>
+        <div class="usertext">
+
+                            <a href="/c:mimingdayo"><span class="username">miming(<wbr>みみん)<wbr></span></a>
+            
+            <div class="fullname">@c:<wbr>mimingdayo<wbr></div>        </div>
+    </div>
+                </div>
+            <div class="tw-search-result-row-user-desc">韓国人です。日本語勉強はじめたのは10年くらい前からで
+留学はしたことありません。
+まぁ気ままにやります。
+ヾ(⌒(_*&#039;ω&#039;*)_</div>
+            <div class="tw-search-result-row-user-tags">
+                <a href="/search/tag/%E6%AD%8C" class="tag tag-mini tag-info">歌</a> <a href="/search/tag/VOCALOID" class="tag tag-mini tag-info">VOCALOID</a> <a href="/search/tag/J-POP" class="tag tag-mini tag-info">J-POP</a> <a href="/search/tag/%E6%97%A5%E6%9C%AC%E8%AA%9E" class="tag tag-mini tag-info">日本語</a> <a href="/search/tag/%E9%9F%93%E5%9B%BD%E4%BA%BA" class="tag tag-mini tag-info">韓国人</a>             </div>
+        </div>
+    </div>
+                    </div>
+                
+            
+
+            <div class="tw-search-result-next-tags">
+                                    <span class="tw-search-result-next-tags-title">Find From Live Tags</span>
+                    <a href="/search/tag/RTPDenVIVO" class="tag  tag-info">RTPDenVIVO</a> <a href="/search/tag/Apex" class="tag  tag-info">Apex</a> <a href="/search/tag/gdgd" class="tag  tag-info">gdgd</a> <a href="/search/tag/DBD" class="tag  tag-info">DBD</a> <a href="/search/tag/PUBG" class="tag  tag-info">PUBG</a> <a href="/search/tag/ApexLegends" class="tag  tag-info">ApexLegends</a> <a href="/search/tag/YouTube" class="tag  tag-info">YouTube</a> <a href="/search/tag/VALORANT" class="tag  tag-info">VALORANT</a> <a href="/search/tag/%E6%AD%8C" class="tag  tag-info">歌</a> <a href="/search/tag/v%E7%B3%BB" class="tag  tag-info">v系</a> <a href="/search/tag/AAA" class="tag  tag-info">AAA</a> <a href="/search/tag/MW" class="tag  tag-info">MW</a> <a href="/search/tag/WoWs" class="tag  tag-info">WoWs</a> <a href="/search/tag/WarThunder" class="tag  tag-info">WarThunder</a> <a href="/search/tag/%E7%8A%AC" class="tag  tag-info">犬</a> <a href="/search/tag/%EC%86%8C%ED%86%B5" class="tag  tag-info">소통</a> <a href="/search/tag/%ED%9E%99%ED%95%A9" class="tag  tag-info">힙합</a>             </div>
+
+        </div>
+
+
+        <div id="navigation">
+                <div id="navibox" class="tw-basic-page-right-column">
+                <h2>Live Tags</h2>
+                <div class="box">
+                    <p class="taglist">
+                        <a href="/search/tag/RTPDenVIVO" class="tag  tag-info">RTPDenVIVO</a> <a href="/search/tag/Apex" class="tag  tag-info">Apex</a> <a href="/search/tag/DBD" class="tag  tag-info">DBD</a> <a href="/search/tag/PUBG" class="tag  tag-info">PUBG</a> <a href="/search/tag/ApexLegends" class="tag  tag-info">ApexLegends</a> <a href="/search/tag/YouTube" class="tag  tag-info">YouTube</a> <a href="/search/tag/VALORANT" class="tag  tag-info">VALORANT</a> <a href="/search/tag/%E6%AD%8C" class="tag  tag-info">歌</a> <a href="/search/tag/v%E7%B3%BB" class="tag  tag-info">v系</a> <a href="/search/tag/AAA" class="tag  tag-info">AAA</a> <a href="/search/tag/MW" class="tag  tag-info">MW</a> <a href="/search/tag/WoWs" class="tag  tag-info">WoWs</a> <a href="/search/tag/WarThunder" class="tag  tag-info">WarThunder</a> <a href="/search/tag/%E7%8A%AC" class="tag  tag-info">犬</a> <a href="/search/tag/%EC%86%8C%ED%86%B5" class="tag  tag-info">소통</a> <a href="/search/tag/%ED%9E%99%ED%95%A9" class="tag  tag-info">힙합</a> <a href="/search/tag/gdgd" class="tag  tag-info">gdgd</a>                     </p>
+                </div>
+                            </div>
+            </div>
+
+
+        <div id="ad_footer" style="padding:10px 1px;clear:left;width: auto;"><script type="text/javascript">
+    window.addEventListener("DOMContentLoaded", function () {
+        var tag = new Moiad.SingleAdTag();
+        tag.setElementId("ad_footer");
+        tag.setPlaceName("tcx_adfooter_pc_300x250");
+        adLoader.activate(tag); 
+    });    
+</script></div><div style="clear:both"></div>
+    </div>
+
+</div>
+
+
+<footer class="tw-footer">
+    <div class="tw-footer-container">
+        <section class="tw-footer-application-area">
+            <div class="tw-footer-column">
+                <h3 class="tw-footer-column-title">Viewer App</h3>
+                <div class="tw-footer-application">
+                    <div class="tw-footer-application-icon"><img src="/img/top/icon_application01.png" alt="Viewer App"></div>
+                    <ul class="tw-footer-application-list">
+                        <li class="tw-footer-application-list-item"><a href="https://itunes.apple.com/app/twitcasting-viewer/id365235835?mt=8"><img
+                                        src="/img/badge_app_store.png" alt="App Store"></a></li>
+                        <li class="tw-footer-application-list-item"><a href="https://play.google.com/store/apps/details?id=com.sidefeed.TCViewer"><img
+                                        src="/img/badge_android_market.png" alt="Google Play"></a></li>
+                    </ul>
+                </div>
+            </div>
+            <div class="tw-footer-column">
+                <h3 class="tw-footer-column-title">Live App</h3>
+                <div class="tw-footer-application">
+                    <div class="tw-footer-application-icon"><img src="/img/top/icon_application02.png" alt="Live App"></div>
+                    <ul class="tw-footer-application-list">
+                        <li class="tw-footer-application-list-item"><a href="https://itunes.apple.com/app/twitcasting-live/id352756035?mt=8"><img
+                                        src="/img/badge_app_store.png" alt="App Store"></a></li>
+                        <li class="tw-footer-application-list-item"><a href="https://play.google.com/store/apps/details?id=com.sidefeed.TCLive"><img
+                                        src="/img/badge_android_market.png" alt="Google Play"></a></li>
+                    </ul>
+                </div>
+            </div>
+            <div class="tw-footer-column">
+                <h3 class="tw-footer-column-title">TwitCasting Games</h3>
+                <div class="tw-footer-application">
+                    <div class="tw-footer-application-icon"><img src="/img/top/icon_application03.png" alt="TwitCasting Games"></div>
+                    <ul class="tw-footer-application-list">
+                        <li class="tw-footer-application-list-item"><a href="https://itunes.apple.com/app/twitcasting-live/id1444520376?mt=8"><img
+                                        src="/img/badge_app_store.png" alt="App Store"></a></li>
+                        <li class="tw-footer-application-list-item"><a href="https://play.google.com/store/apps/details?id=com.sidefeed.screencas"><img
+                                        src="/img/badge_android_market.png" alt="Google Play"></a></li>
+                    </ul>
+                </div>
+            </div>
+        </section>
+        <section class="tw-footer-navi-area">
+            <div class="tw-footer-column">
+                <div class="tw-footer-row">
+                    <h4 class="tw-footer-column-title"><label
+                                for="tw-footer-navi-visibility">About</label></h4>
+                    <input class="tw-footer-column-visibility" id="tw-footer-navi-visibility" type="checkbox"
+                           checked="checked">
+                    <ul class="tw-footer-navi">
+                        <li><a href="/indexprivacy.php">Privacy Policy</a></li>
+                        <li><a href="/indexlicense.php">Terms of Service</a></li>
+                                                <li><a href="/indexattribution.php">Attribution</a></li>
+                        <li><a href="/indexapi.php">Developer API</a></li>
+                                            </ul>
+                    <span class="tw-global-footer-icon-down"><svg viewBox="0 0 24 24" width="24" height="24" xmlns="http://www.w3.org/2000/svg"><path d="M.531 8.035c-.708-.708-.708-1.911 0-2.62l.39-.388c.707-.708 1.91-.708 2.619 0l8.46 8.46 8.46-8.46c.708-.708 1.912-.708 2.62 0l.389.389c.708.708.708 1.911 0 2.62L12 19.503.531 8.035z" fill="#000" fill-rule="evenodd"/></svg>
+</span>
+                </div>
+
+                            </div>
+                        <div class="tw-footer-column">
+                <h4 class="tw-footer-column-title"><label for="tw-footer-language-visibility">Country</label></h4>
+                <input class="tw-footer-column-visibility" id="tw-footer-language-visibility" type="checkbox"
+                       checked="checked">
+                <ul class="tw-footer-navi tw-footer-lang-list" role="listbox">
+                                            <li>
+                            <a href="http://ja.twitcasting.tv/search/tag/vocaloid?&hl=ja" role="option" aria-selected="false">
+                                <img class="tw-global-header-lang-icon" src="/img/country/jp_icon.png" alt="">
+                                日本語                            </a>
+                        </li>
+                                            <li>
+                            <a href="http://es.twitcasting.tv/search/tag/vocaloid?&hl=es" role="option" aria-selected="false">
+                                <img class="tw-global-header-lang-icon" src="/img/country/mx_icon.png" alt="">
+                                Español                            </a>
+                        </li>
+                                            <li>
+                            <a href="http://pt.twitcasting.tv/search/tag/vocaloid?&hl=pt" role="option" aria-selected="false">
+                                <img class="tw-global-header-lang-icon" src="/img/country/br_icon.png" alt="">
+                                Português                            </a>
+                        </li>
+                                            <li>
+                            <a href="http://en.twitcasting.tv/search/tag/vocaloid?&hl=en" role="option" aria-selected="true">
+                                <img class="tw-global-header-lang-icon" src="/img/country/us_icon.png" alt="">
+                                English                            </a>
+                        </li>
+                                    </ul>
+                <span class="tw-global-footer-icon-down"><svg viewBox="0 0 24 24" width="24" height="24" xmlns="http://www.w3.org/2000/svg"><path d="M.531 8.035c-.708-.708-.708-1.911 0-2.62l.39-.388c.707-.708 1.91-.708 2.619 0l8.46 8.46 8.46-8.46c.708-.708 1.912-.708 2.62 0l.389.389c.708.708.708 1.911 0 2.62L12 19.503.531 8.035z" fill="#000" fill-rule="evenodd"/></svg>
+</span>
+            </div>
+        </section>
+        <section class="tw-footer-copy-area">
+            Copyright (c) 2009-2020 <a href="https://about.moi.st/">Moi Corp.</a> (Moi Labs inc.)
+        </section>
+    </div>
+</footer>
+
+    <!-- Facebook Pixel Code -->
+    <script>
+        !function(f,b,e,v,n,t,s)
+        {if(f.fbq)return;n=f.fbq=function(){n.callMethod?
+            n.callMethod.apply(n,arguments):n.queue.push(arguments)};
+            if(!f._fbq)f._fbq=n;n.push=n;n.loaded=!0;n.version='2.0';
+            n.queue=[];t=b.createElement(e);t.async=!0;
+            t.src=v;s=b.getElementsByTagName(e)[0];
+            s.parentNode.insertBefore(t,s)}(window, document,'script',
+            'https://connect.facebook.net/en_US/fbevents.js');
+        fbq('init', '264399894717032');
+        fbq('track', 'PageView');
+    </script>
+    <noscript><img height="1" width="1" style="display:none"
+                   src="https://www.facebook.com/tr?id=264399894717032&ev=PageView&noscript=1"/></noscript>
+    <!-- End Facebook Pixel Code -->
+
+    <!-- Twitter universal website tag code -->
+    <script>
+        !function(e,t,n,s,u,a){e.twq||(s=e.twq=function(){s.exe?s.exe.apply(s,arguments):s.queue.push(arguments);
+        },s.version='1.1',s.queue=[],u=t.createElement(n),u.async=!0,u.src='//static.ads-twitter.com/uwt.js',
+            a=t.getElementsByTagName(n)[0],a.parentNode.insertBefore(u,a))}(window,document,'script');
+        // Insert Twitter Pixel ID and Standard Event data below
+        twq('init','o3l79');
+        twq('track','PageView');
+    </script>
+    <!-- End Twitter universal website tag code -->
+
+    <!-- Global site tag (gtag.js) - Google Ads: 649663333 -->
+    <script async src="https://www.googletagmanager.com/gtag/js?id=AW-649663333"></script>
+    <script>
+        window.dataLayer = window.dataLayer || [];
+        function gtag(){dataLayer.push(arguments);}
+        gtag('js', new Date());
+
+        gtag('config', 'AW-649663333');
+    </script>
+
+    <!-- LINE Tag Base Code -->
+    <!-- Do Not Modify -->
+    <script>
+        (function(g,d,o){
+            g._ltq=g._ltq||[];g._lt=g._lt||function(){g._ltq.push(arguments)};
+            var h=location.protocol==='https:'?'https://d.line-scdn.net':'http://d.line-cdn.net';
+            var s=d.createElement('script');s.async=1;
+            s.src=o||h+'/n/line_tag/public/release/v1/lt.js';
+            var t=d.getElementsByTagName('script')[0];t.parentNode.insertBefore(s,t);
+        })(window, document);
+        _lt('init', {
+            customerType: 'lap',
+            tagId: 'c0619881-5f46-411d-b23a-a077e3506417'
+        });
+        _lt('send', 'pv', ['c0619881-5f46-411d-b23a-a077e3506417']);
+    </script>
+    <noscript>
+        <img height="1" width="1" style="display:none"
+             src="https://tr.line.me/tag.gif?c_t=lap&t_id=c0619881-5f46-411d-b23a-a077e3506417&e=pv&noscript=1" />
+    </noscript>
+    <!-- End LINE Tag Base Code -->
+    <script async src="https://s.yimg.jp/images/listing/tool/cv/ytag.js"></script>
+    <script>
+        window.yjDataLayer = window.yjDataLayer || [];
+        function ytag() { yjDataLayer.push(arguments); }
+        ytag({"type":"ycl_cookie"});
+    </script>
+    
+            
+
+
+</body>
+</html>

tasks/twitcasting/twitcasting_task1/main.go

@@ -0,0 +1,5 @@
+package main
+
+func main() {
+	Execute()
+}

tasks/twitcasting/twitcasting_task1/main_test.go

@@ -0,0 +1,25 @@
+package main
+
+import (
+	"net/url"
+
+	"testing"
+)
+
+func Test(t *testing.T) {
+	rawurl := "https://twitcasting.tv/你好"
+	u, _ := url.Parse(rawurl)
+	t.Error(u.EscapedPath())
+	t.Error(u.String())
+}
+
+func TestUpdateTime(t *testing.T) {
+	// streamer := &intimate.Streamer{}
+	// streamer.Uid = 420153
+	// streamer.UpdateTime = sql.NullTime{Time: time.Now(), Valid: true}
+	// estore.Update(streamer, "update_time", streamer.UpdateTime)
+}
+
+func TestMain(t *testing.T) {
+
+}

tasks/twitcasting/twitcasting_task1/twitcasting.go

@@ -0,0 +1,140 @@
+package main
+
+import (
+	"database/sql"
+	"encoding/json"
+	"intimate"
+	"log"
+	"net/url"
+	"time"
+
+	"github.com/474420502/extractor"
+	"github.com/474420502/focus/compare"
+	"github.com/474420502/focus/tree/heap"
+	"github.com/474420502/requests"
+)
+
+// sstore  源存储实例, 为存储源数据的实现. 表格具体参考sql/intimate_source.sql
+var sstore *intimate.StoreSource = intimate.NewStoreSource(string(intimate.STTwitcasting))
+
+// estore 解析存储连接实例
+var estore *intimate.StoreExtractor = intimate.NewStoreExtractor()
+
+type SearchProfile struct {
+	UserName string   `exp:".//span[@class='username']" method:"Text"`
+	UserId   string   // `exp:".//span[@class='fullname']" method:"Text"`
+	LiveUrl  string   `exp:".//div[@class='usertext']/a[@href]" method:"AttributeValue,href"`
+	Tag      []string `exp:".//a[contains(@class, 'tag tag-mini')]" method:"Text"`
+	TagUrl   []string `exp:".//a[contains(@class, 'tag tag-mini')]" method:"AttributeValue,href"`
+}
+
+func Execute() {
+	homeurl := "https://twitcasting.tv"
+	searchurl := "https://twitcasting.tv/rankingindex.php"
+	queuedict := make(map[string]bool)
+	queue := heap.New(compare.String)
+	queue.Put(searchurl)
+	queuedict[searchurl] = true
+	ses := requests.NewSession()
+	ses.Config().SetTimeout(15)
+
+	var surl interface{}
+	var ok bool
+
+	ps := intimate.NewPerfectShutdown()
+
+	for surl, ok = queue.Pop(); ok && !ps.IsClose(); surl, ok = queue.Pop() {
+		u, err := url.Parse(surl.(string))
+		if err != nil {
+			log.Println(err)
+			continue
+		}
+
+		resp, err := ses.Get(u.String()).Execute()
+		if err != nil {
+			log.Println(err)
+			log.Println(u.String(), surl)
+			continue
+			// log.Panic(err)
+		}
+
+		etor := extractor.ExtractHtml(resp.Content())
+		result, err := etor.XPaths("//p[@class='taglist']/a[contains(@class, 'tag')]/@href")
+		if err != nil {
+			panic(err)
+		}
+
+		for _, href := range result.GetTexts() {
+
+			wurl := homeurl + href
+			if ok := queuedict[wurl]; !ok {
+				log.Println(wurl)
+				sl := &intimate.StreamerList{}
+				sl.Platform = intimate.Ptwitcasting
+				sl.Url = wurl
+				sl.Operator = 0
+				sl.UpdateInterval = 120
+				sl.UpdateTime = time.Now()
+
+				estore.InsertStreamerList(sl)
+
+				queue.Put(wurl)
+				queuedict[wurl] = true
+			}
+		}
+
+		xps, err := etor.XPaths("//div[@class='tw-search-result-row']")
+		if err != nil {
+			log.Println(surl, err)
+			continue
+		}
+
+		var splist = xps.ForEachObjectByTag(SearchProfile{})
+		for _, isp := range splist {
+			sp := isp.(*SearchProfile)
+			if sp.LiveUrl == "" {
+				continue
+			}
+
+			sp.UserId = sp.LiveUrl[1:]
+			for i := 0; i < len(sp.TagUrl); i++ {
+				wurl := homeurl + sp.TagUrl[i]
+				sp.TagUrl[i] = wurl
+				if ok := queuedict[wurl]; !ok {
+					sl := &intimate.StreamerList{}
+					sl.Platform = intimate.Ptwitcasting
+					sl.Url = wurl
+					sl.Operator = 0
+					sl.UpdateInterval = 120
+					sl.UpdateTime = time.Now()
+					estore.InsertStreamerList(sl)
+
+					queue.Put(wurl)
+					queuedict[wurl] = true
+				}
+			}
+			// log.Println(sp.(SearchProfile))
+		}
+
+		log.Println("find user:", len(splist))
+		for _, isp := range splist {
+			sp := isp.(*SearchProfile)
+			// log.Println(sp)
+			streamer := &intimate.Streamer{}
+			streamer.Platform = intimate.Ptwitcasting
+			streamer.LiveUrl = sql.NullString{String: sp.LiveUrl, Valid: true}
+			if btags, err := json.Marshal(sp.Tag); err != nil {
+				log.Println(err)
+			} else {
+				streamer.Tags = btags
+			}
+			streamer.UpdateInterval = 120
+			streamer.UpdateTime = sql.NullTime{Time: time.Now(), Valid: true}
+			streamer.UserName = sql.NullString{String: sp.UserName, Valid: true}
+			streamer.UserId = sp.UserId
+			estore.InsertStreamer(streamer)
+		}
+
+		log.Println("finish remain", queue.Size())
+	}
+}

tasks/twitch/twitch_task1/.gitignore

@@ -0,0 +1,2 @@
+twitch_task1
+log

tasks/twitch/twitch_task1/main.go

@@ -0,0 +1,5 @@
+package main
+
+func main() {
+	Execute()
+}

tasks/twitch/twitch_task1/task_twitch.go

@@ -0,0 +1,108 @@
+package main
+
+import (
+	"database/sql"
+	"intimate"
+	"log"
+	"time"
+
+	"github.com/tebeka/selenium"
+)
+
+// sstore  源存储实例, 为存储源数据的实现. 表格具体参考sql/intimate_source.sql
+var sstore *intimate.StoreSource = intimate.NewStoreSource(string(intimate.STTwitch))
+
+// estore 解析存储连接实例
+var estore *intimate.StoreExtractor = intimate.NewStoreExtractor()
+
+// 获取类型的所有频道链接
+
+// Execute 执行任务
+func Execute() {
+	var err error
+	wd := intimate.GetChromeDriver(3030)
+	ps := intimate.NewPerfectShutdown()
+
+	weburl := "https://www.twitch.tv/directory?sort=VIEWER_COUNT"
+	err = wd.Get(weburl)
+	if err != nil {
+		panic(err)
+	}
+
+	cardCondition := func(wd selenium.WebDriver) (bool, error) {
+		elements, err := wd.FindElements(selenium.ByXPATH, "//span/a[contains(@data-a-target,'card-') and @href]")
+		if err != nil {
+			return false, err
+		}
+		return len(elements) > 0, nil
+	}
+	wd.WaitWithTimeout(cardCondition, time.Second*15)
+	time.Sleep(time.Second)
+
+	e, err := wd.FindElement(selenium.ByXPATH, "//button[@data-a-target='browse-sort-menu']")
+	if err != nil {
+		panic(err)
+	}
+	e.Click()
+
+	var hrefs map[string]bool = make(map[string]bool)
+	var delayerror = 5
+	for i := 0; i <= 200; i++ {
+		cards, err := wd.FindElements(selenium.ByXPATH, "//span/a[contains(@data-a-target,'card-') and @href]")
+		if err != nil {
+			log.Println(err)
+			break
+		}
+
+		if len(hrefs) == 0 {
+			delayerror--
+			if delayerror <= 0 {
+				break
+			}
+		} else {
+			delayerror = 5
+		}
+
+		for ii := 0; ii < 10; ii++ {
+			for _, card := range cards {
+				href, err := card.GetAttribute("href")
+				if err != nil {
+					log.Println(href, err)
+					continue
+				} else {
+					hrefs[href] = true
+				}
+			}
+			break
+		}
+
+		if ps.IsClose() {
+			break
+		}
+
+		if len(cards) > 10 {
+			log.Println(len(cards))
+			wd.ExecuteScript(`items = document.evaluate("//div[@data-target='directory-page__card-container']/../self::div[@data-target and @style]", document, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null); 
+		for (var i = 0; i < items.snapshotLength - 10; i++) { item = items.snapshotItem(i); item.remove() ;};`, nil)
+		}
+		time.Sleep(time.Millisecond * 200)
+		wd.KeyDown(selenium.EndKey)
+		time.Sleep(time.Millisecond * 200)
+		wd.KeyUp(selenium.EndKey)
+		time.Sleep(time.Millisecond * 2500)
+	}
+
+	for href := range hrefs {
+
+		// TODO: Save href
+		source := &intimate.Source{}
+		source.Source = sql.NullString{String: href, Valid: true}
+		source.Operator = 0
+		source.Target = intimate.TTwitchChannel
+		source.Url = weburl
+		sstore.Insert(source)
+	}
+
+	log.Println("hrefs len:", len(hrefs))
+	sstore.Deduplicate(intimate.TTwitchChannel, "source")
+}

tasks/twitch/twitch_task1/task_twitch_test.go

@@ -0,0 +1,13 @@
+package main
+
+import (
+	"testing"
+)
+
+func estCase1(t *testing.T) {
+	Execute()
+}
+
+func TestLiveUrl(t *testing.T) {
+
+}

tasks/twitch/twitch_task2/.gitignore

@@ -0,0 +1,2 @@
+twitch_task2
+log

tasks/twitch/twitch_task2/main.go

@@ -0,0 +1,6 @@
+package main
+
+func main() {
+
+	Execute()
+}

tasks/twitch/twitch_task2/task_twitch.go

@@ -0,0 +1,172 @@
+package main
+
+import (
+	"database/sql"
+	"encoding/json"
+	"intimate"
+	"log"
+	"regexp"
+	"time"
+
+	"github.com/tebeka/selenium"
+)
+
+// sstore  源存储实例, 为存储源数据的实现. 表格具体参考sql/intimate_source.sql
+var sstore *intimate.StoreSource = intimate.NewStoreSource(string(intimate.STTwitch))
+
+// estore 解析存储连接实例
+var estore *intimate.StoreExtractor = intimate.NewStoreExtractor()
+
+// 获取类型的所有频道链接
+
+// Execute 执行任务
+func Execute() {
+	// DELETE FROM source_twitch WHERE uid NOT IN (SELECT MAX(s.uid) FROM (SELECT uid, source FROM source_twitch ) s GROUP BY s.source) ;
+	//article//a[@data-a-target='preview-card-title-link']
+
+	wd := intimate.GetChromeDriver(3030)
+	defer wd.Quit()
+	ps := intimate.NewPerfectShutdown()
+	counter := intimate.NewCounter()
+	counter.SetMaxLimit(100)
+	counter.SetMaxToDo(func(olist ...interface{}) error {
+		owd := olist[0].(*selenium.WebDriver)
+		if err := (*owd).Quit(); err != nil {
+			log.Println(err)
+		}
+		*owd = intimate.GetChromeDriver(3030)
+		return nil
+	}, &wd)
+
+	for !ps.IsClose() {
+
+		var err error
+		sourceChannel, err := sstore.Pop(intimate.TTwitchChannel)
+		if err != nil {
+			panic(err)
+		}
+
+		weburl := sourceChannel.Source.String + "?sort=VIEWER_COUNT"
+		err = wd.Get(weburl)
+		if err != nil {
+			log.Println(err)
+			sstore.UpdateError(sourceChannel, err)
+			time.Sleep(time.Second * 10)
+			continue
+		}
+
+		wd.WaitWithTimeout(func(wd selenium.WebDriver) (bool, error) {
+			_, err := wd.FindElement(selenium.ByXPATH, "(//div/p[@class=''])[last()]")
+			if err != nil {
+				return false, err
+			}
+			return true, nil
+		}, time.Second*10)
+
+		btn, err := wd.FindElement(selenium.ByXPATH, "//button[@data-a-target='browse-sort-menu']")
+		if err != nil {
+			log.Println(err)
+			continue
+		}
+		btn.Click()
+
+		var elements []selenium.WebElement
+		var liveurls = 0
+		var delayerror = 2
+		for i := 0; i < 200 && !ps.IsClose(); i++ {
+			elements, err = wd.FindElements(selenium.ByXPATH, "(//div/p[@class=''])[last()]")
+			if err != nil {
+				log.Println(err)
+				break
+			}
+			time.Sleep(time.Millisecond * 200)
+			wd.KeyDown(selenium.EndKey)
+			time.Sleep(time.Millisecond * 200)
+			wd.KeyUp(selenium.EndKey)
+			time.Sleep(time.Millisecond * 2000)
+			if len(elements) == liveurls {
+				delayerror--
+				if delayerror <= 0 {
+					break
+				}
+			} else {
+				delayerror = 2
+			}
+			liveurls = len(elements)
+		}
+		articles, err := wd.FindElements(selenium.ByXPATH, "//article")
+		if err != nil {
+			log.Println(err)
+			continue
+		}
+
+		for _, article := range articles {
+
+			e, err := article.FindElement(selenium.ByXPATH, ".//a[@data-a-target='preview-card-title-link' and @href]")
+			if err != nil {
+				log.Println(err)
+				continue
+			}
+
+			href, err := e.GetAttribute("href")
+			if err != nil {
+				log.Println(err)
+				continue
+			}
+
+			btns, err := article.FindElements(selenium.ByXPATH, ".//div[@class='tw-full-width tw-inline-block']//button")
+			if err != nil {
+				log.Println(err)
+				continue
+			}
+
+			var tags []string
+			for _, btn := range btns {
+				tag, err := btn.GetAttribute("data-a-target")
+				if err == nil {
+					tags = append(tags, tag)
+				}
+			}
+
+			streamer := &intimate.Streamer{}
+
+			matches := regexp.MustCompile(`https://www.twitch.tv/(\w+)`).FindStringSubmatch(href)
+			if len(matches) == 2 {
+				streamer.UserId = matches[1]
+			} else {
+				log.Println(href)
+				continue
+			}
+
+			jtags, err := json.Marshal(tags)
+			if err != nil {
+				log.Println(err)
+			} else {
+				streamer.Tags = jtags
+			}
+
+			streamer.Platform = intimate.Ptwitch
+
+			updateUrl := make(map[string]string)
+			updateUrl["live"] = href
+			streamer.LiveUrl = sql.NullString{String: href, Valid: true}
+			data, err := json.Marshal(updateUrl)
+			if err != nil {
+				log.Println(err)
+				continue
+			}
+			streamer.UpdateUrl = data
+			streamer.Operator = 0
+			if estore.InsertStreamer(streamer) {
+				// log.Println("streamer update tags", streamer.Uid, tags)
+				estore.Update(streamer, "Tags", streamer.Tags)
+			}
+		}
+		log.Println("streamer find", len(articles))
+		if len(articles) == 0 {
+			sourceChannel.Operator = 5
+			sstore.UpdateOperator(sourceChannel)
+		}
+		counter.AddWithReset(1)
+	}
+}

tasks/twitch/twitch_task2/task_twitch_test.go

@@ -0,0 +1,7 @@
+package main
+
+import "testing"
+
+func estMain(t *testing.T) {
+	main()
+}

utils.go

@@ -1,8 +1,19 @@
 package intimate
 
 import (
+	"fmt"
 	"log"
+	"os"
+	"os/signal"
+	"runtime"
+	"strconv"
+	"strings"
+	"sync/atomic"
+	"syscall"
 	"time"
+
+	"github.com/tebeka/selenium"
+	"github.com/tebeka/selenium/chrome"
 )
 
 var zeroTime time.Time
@@ -17,6 +28,35 @@ func init() {
 
 }
 
+// ParseNumber 去逗号解析数字
+func ParseNumber(num string) (int64, error) {
+	num = strings.Trim(num, " ")
+	num = strings.ReplaceAll(num, ",", "")
+	return strconv.ParseInt(num, 10, 64)
+}
+
+// ParseNumberEx 解析带字符的数字
+func ParseNumberEx(num string) (float64, error) {
+	num = strings.Trim(num, " ")
+	num = strings.ReplaceAll(num, ",", "")
+	last := num[len(num)-1]
+	factor := 1.0
+	switch {
+	case last == 'k' || last == 'K':
+		factor = 1000.0
+		num = num[0 : len(num)-1]
+	case last == 'm' || last == 'M':
+		factor = 1000000.0
+		num = num[0 : len(num)-1]
+	}
+	i, err := strconv.ParseFloat(num, 64)
+	if err != nil {
+		return 0, err
+	}
+
+	return i * factor, nil
+}
+
 // ParseDuration time to duration eg: 1:40:00 -> time.Duration
 func ParseDuration(dt string) (time.Duration, error) {
 
@@ -37,8 +77,184 @@ func ParseDuration(dt string) (time.Duration, error) {
 
 	tdt, err := time.Parse("15:04:05", string(parse))
 	if err != nil {
-
 		return time.Duration(0), err
 	}
 	return tdt.Sub(zeroTime), nil
 }
+
+func GetChromeDriver(port int) selenium.WebDriver {
+	var err error
+	caps := selenium.Capabilities{"browserName": "chrome"}
+
+	chromecaps := chrome.Capabilities{}
+	for _, epath := range []string{"../../../crx/myblock.crx", "../../crx/myblock.crx"} {
+		_, err := os.Stat(epath)
+		if err == nil {
+			err := chromecaps.AddExtension(epath)
+			if err != nil {
+				panic(err)
+			}
+			break
+		}
+	}
+
+	if proxy := os.Getenv("chrome_proxy"); proxy != "" {
+		log.Println("proxy-server", proxy)
+		chromecaps.Args = append(chromecaps.Args, "--proxy-server="+proxy)
+	}
+
+	if proxy := os.Getenv("pac_proxy"); proxy != "" {
+		log.Println("--proxy-pac-url=" + proxy)
+		chromecaps.Args = append(chromecaps.Args, "--proxy-pac-url="+proxy)
+	}
+
+	// chromecaps.Args = append(chromecaps.Args, "--proxy-pac-url=http://127.0.0.1:1081/pac")
+	chromecaps.Args = append(chromecaps.Args, "--disk-cache-dir=/tmp/chromedriver-cache")
+	chromecaps.Args = append(chromecaps.Args, "--disable-gpu", "--disable-images", "--start-maximized", "--disable-infobars")
+	// chromecaps.Args = append(chromecaps.Args, "--headless")
+	chromecaps.Args = append(chromecaps.Args, "--no-sandbox")
+	chromecaps.Args = append(chromecaps.Args, "--disable-dev-shm-usage", "--mute-audio", "--safebrowsing-disable-auto-update")
+
+	chromecaps.ExcludeSwitches = append(chromecaps.ExcludeSwitches, "enable-automation")
+	caps.AddChrome(chromecaps)
+	_, err = selenium.NewChromeDriverService("/usr/bin/chromedriver", port)
+	if err != nil {
+		panic(err)
+	}
+	wd, err := selenium.NewRemote(caps, fmt.Sprintf("http://localhost:%d/wd/hub", port))
+	if err != nil {
+		panic(err)
+	}
+	runtime.SetFinalizer(wd, func(obj interface{}) {
+		if err := obj.(selenium.WebDriver).Close(); err != nil {
+			log.Println(err)
+		}
+		if err := obj.(selenium.WebDriver).Quit(); err != nil {
+			log.Println(err)
+		}
+	})
+	wd.ExecuteScript("windows.navigator.webdriver = undefined", nil)
+	if err != nil {
+		panic(err)
+	}
+
+	return wd
+}
+
+// PerfectShutdown 完美关闭程序
+type PerfectShutdown struct {
+	loop int32
+}
+
+// NewPerfectShutdown 创建完美关闭程序
+func NewPerfectShutdown() *PerfectShutdown {
+	ps := &PerfectShutdown{}
+	ps.loop = 1
+
+	go func() {
+		signalchan := make(chan os.Signal)
+		signal.Notify(signalchan, syscall.SIGINT, syscall.SIGKILL, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGSTOP)
+		log.Println("accept stop command:", <-signalchan)
+		atomic.StoreInt32(&ps.loop, 0)
+	}()
+
+	return ps
+}
+
+// IsClose 判断是否要关闭
+func (ps *PerfectShutdown) IsClose() bool {
+	return atomic.LoadInt32(&ps.loop) == 0
+}
+
+type Counter struct {
+	dcount   int
+	count    int
+	maxLimit int
+	minLimit int
+
+	minobj       []interface{}
+	maxobj       []interface{}
+	maxLimitToDo func(obj ...interface{}) error
+	minLimitToDo func(obj ...interface{}) error
+}
+
+func NewCounter() *Counter {
+	c := &Counter{}
+	return c
+}
+
+// SetDefault 设置默认值
+func (c *Counter) SetDefault(n int) {
+	c.dcount = n
+}
+
+// Reset 最置count为defaultCount值
+func (c *Counter) Reset() {
+	c.count = c.dcount
+}
+
+// SetCount 设置count到最大值的时候执行do函数
+func (c *Counter) SetCount(count int) {
+	c.count = count
+}
+
+// GetCount 设置count到最大值的时候执行do函数
+func (c *Counter) GetCount() int {
+	return c.count
+}
+
+// SetMinLimit 设置最小限制
+func (c *Counter) SetMinLimit(n int) {
+	c.minLimit = n
+}
+
+// SetMaxLimit 设置最大限制
+func (c *Counter) SetMaxLimit(n int) {
+	c.maxLimit = n
+}
+
+// SetMaxToDo 设置count到最大值的时候执行do函数
+func (c *Counter) SetMaxToDo(do func(obj ...interface{}) error, obj ...interface{}) {
+	c.maxLimitToDo = do
+	c.maxobj = obj
+}
+
+// SetMinToDo 设置count到最小值的时候执行do函数
+func (c *Counter) SetMinToDo(do func(obj ...interface{}) error, obj ...interface{}) {
+	c.minLimitToDo = do
+	c.minobj = obj
+}
+
+// AddWithReset 操作 count 默认值为0, 当触发限制时, 重置为默认值
+func (c *Counter) AddWithReset(n int) error {
+	c.count += n
+	if c.maxLimitToDo != nil {
+		if c.count >= c.maxLimit {
+			defer c.Reset()
+			return c.maxLimitToDo(c.maxobj...)
+		}
+	}
+	if c.minLimitToDo != nil {
+		if c.count <= c.minLimit {
+			defer c.Reset()
+			return c.minLimitToDo(c.minobj...)
+		}
+	}
+	return nil
+}
+
+// Add 操作 count 默认值为0
+func (c *Counter) Add(n int) error {
+	c.count += n
+	if c.maxLimitToDo != nil {
+		if c.count >= c.maxLimit {
+			return c.maxLimitToDo(c.maxobj...)
+		}
+	}
+	if c.minLimitToDo != nil {
+		if c.count <= c.minLimit {
+			return c.minLimitToDo(c.minobj...)
+		}
+	}
+	return nil
+}

xvfb.sh

@@ -0,0 +1 @@
+screen -dmS xvfb-99  Xvfb :99 -screen 0 1280x720x24 -ac -nolisten tcp -dpi 96 +extension RANDR -nolisten tcp