add: 优雅停机

extractor/openrec_extractor/.gitignore

@@ -0,0 +1,4 @@
+*.html
+log
+screenlog.*
+openrec_extractor

extractor/openrec_extractor/main.go

@@ -0,0 +1,16 @@
+package main
+
+/*
+  `uid`           varchar(36)  NOT NULL,
+  `platform`      varchar(255) NOT NULL,
+  `anchor_id`     varchar(255) NOT NULL,
+  `anchor_name`   varchar(255) NOT NULL,
+  `live_url`      text,
+  `channel`       varchar(128) DEFAULT NULL, // 没有分类
+  `show_type`     varchar(255) DEFAULT NULL,
+*/
+
+func main() {
+	oe := &OpenrecExtractor{}
+	oe.Execute()
+}

extractor/openrec_extractor/openrec_extractor.go

@@ -0,0 +1,244 @@
+package main
+
+import (
+	"database/sql"
+	"encoding/json"
+	"intimate"
+	"log"
+	"os"
+	"os/signal"
+	"regexp"
+	"strconv"
+	"strings"
+	"sync/atomic"
+	"syscall"
+	"time"
+
+	"github.com/tidwall/gjson"
+)
+
+// OpenrecExtractor 提取方法
+type OpenrecExtractor struct {
+	user       *intimate.ExtractorSource
+	userLive   *intimate.ExtractorSource
+	supporters *intimate.ExtractorSource
+}
+
+func (oe *OpenrecExtractor) Execute() {
+
+	var loop int32 = 1
+
+	go func() {
+		signalchan := make(chan os.Signal)
+		signal.Notify(signalchan, syscall.SIGKILL, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGSTOP)
+		log.Println("accept stop command:", <-signalchan)
+		atomic.StoreInt32(&loop, 0)
+	}()
+
+	collect := intimate.NewExtractorStore()
+	store := intimate.NewSourceStore("source_openrec")
+	var lasterr error = nil
+
+	for atomic.LoadInt32(&loop) > 0 {
+
+		source, err := store.Pop(string(intimate.TTOpenrecRanking), 100)
+		if err != nil {
+			if err != lasterr {
+				log.Println(err, lasterr)
+				lasterr = err
+			}
+			time.Sleep(time.Second * 2)
+			continue
+		}
+
+		source.SetOperator(int32(intimate.OperatorError))
+		anchorId := source.GetSource().String
+
+		ai := &intimate.AnchorInfo{}
+		ai.SetAnchorId(anchorId)
+		ai.SetPlatform(string(intimate.Popenrec))
+
+		sdata := source.GetExt().([]byte)
+		if gjson.ValidBytes(sdata) {
+			result := gjson.ParseBytes(sdata)
+			datamap := result.Map()
+
+			oe.user = intimate.NewExtractorSource(datamap["user"])
+			oe.user.CreateExtractor()
+
+			oe.userLive = intimate.NewExtractorSource(datamap["user_live"])
+			oe.userLive.CreateExtractor()
+
+			oe.supporters = intimate.NewExtractorSource(datamap["supporters"])
+
+			clog := &intimate.CollectLog{}
+
+			log.Println(anchorId)
+
+			oe.extractFollowers(clog)
+			oe.extractAnchorName(ai)
+			oe.extractViewsAndLiveStreaming(clog)
+			oe.extractGiversAndGratuity(clog)
+			oe.extractLive(clog)
+			oe.extractTags(clog)
+
+			ai.Set("UpdateTime", source.GetUpdateTime())
+
+			LiveUrl := "https://www.openrec.tv/live/" + anchorId
+			ai.Set("LiveUrl", sql.NullString{String: LiveUrl, Valid: true})
+
+			Uid, err := collect.InsertAnchorInfo(ai)
+			if err != nil {
+				log.Println(err)
+				source.SetErrorMsg(sql.NullString{String: err.Error(), Valid: true})
+				store.UpdateOperator(source)
+				return
+			}
+
+			clog.Set("Uid", Uid)
+			clog.Set("Platform", string(intimate.Popenrec))
+			clog.Set("AnchorId", anchorId)
+			clog.Set("UpdateTime", source.GetUpdateTime())
+
+			if err = collect.InsertCollectLog(clog); err != nil {
+				source.SetErrorMsg(sql.NullString{String: err.Error(), Valid: true})
+				store.UpdateOperator(source)
+				return
+			}
+
+			source.SetOperator(int32(intimate.OperatorExtractorOK))
+			store.UpdateOperator(source)
+		} else {
+			log.Println("data is not json:\n", string(sdata))
+		}
+	}
+}
+
+func (oe *OpenrecExtractor) extractFollowers(clog intimate.ISet) {
+	extractor := oe.user.GetExtractor()
+	xp, err := extractor.XPathResult("//p[@class='c-global__user__count__row__right js-userCountFollowers']/text()")
+	if err != nil {
+		log.Println(err)
+	}
+	if !xp.NodeIter().Next() {
+		log.Println("不存在粉丝数")
+	}
+
+	followers := strings.ReplaceAll(xp.String(), ",", "")
+	followersInt, err := strconv.ParseInt(followers, 10, 64)
+	if err != nil {
+		log.Println(err)
+	}
+
+	clog.Set("Followers", sql.NullInt64{Int64: followersInt, Valid: true})
+}
+
+func (oe *OpenrecExtractor) extractAnchorName(ai intimate.ISet) {
+	extractor := oe.user.GetExtractor()
+	xp, err := extractor.XPathResult("//p[@class='c-global__user__profile__list__name__text official-icon--after']/text()")
+	if xp.NodeIter().Next() {
+		anchorName := xp.String()
+		ai.Set("AnchorName", anchorName)
+	} else {
+		log.Println(err)
+	}
+}
+
+func (oe *OpenrecExtractor) extractViewsAndLiveStreaming(clog intimate.ISet) {
+	extractor := oe.user.GetExtractor()
+	// c-contents
+	xp, err := extractor.XPathResult("//ul[@class='c-contents']//p[@class='c-thumbnailVideo__footer__liveCount']/text()")
+	if err != nil {
+		log.Println(err)
+	}
+	if xp.NodeIter().Next() {
+		views := regexp.MustCompile(`[0-9,]+`).FindString(xp.String())
+		views = strings.ReplaceAll(views, ",", "")
+		viewsint, err := strconv.Atoi(views)
+		if err != nil {
+			log.Println(err)
+		}
+
+		clog.Set("Views", sql.NullInt64{Int64: int64(viewsint), Valid: true})
+		clog.Set("IsLiveStreaming", int32(1))
+	}
+}
+
+func (oe *OpenrecExtractor) extractGiversAndGratuity(clog intimate.ISet) {
+	// extractor := oe.user.GetExtractor()
+	giverjson := oe.supporters.GetSource()
+	var givers []interface{}
+	var gratuity int64 = 0
+
+	for _, v := range giverjson.Array() {
+		giverSource := gjson.Parse(v.String())
+		for _, item := range giverSource.Get("data.items").Array() {
+			givers = append(givers, item.Map())
+			gratuity += item.Get("total_yells").Int()
+		}
+	}
+
+	giversbytes, err := json.Marshal(givers)
+	if err != nil {
+		log.Println(err)
+		clog.Set("ErrorMsg", sql.NullString{String: err.Error(), Valid: true})
+	} else {
+		clog.Set("Giver", giversbytes)
+	}
+
+	clog.Set("Gratuity", sql.NullInt64{Int64: gratuity, Valid: true})
+}
+
+func (oe *OpenrecExtractor) extractLive(clog intimate.ISet) {
+	extractor := oe.userLive.GetExtractor()
+	mathes := regexp.MustCompile("MovieTitle__Title[^>]+>(.{1,50})</h1>").FindStringSubmatch(oe.userLive.GetSource().Str)
+	if len(mathes) == 2 {
+
+		clog.Set("LiveTitle", sql.NullString{String: mathes[1], Valid: true})
+
+		content, err := extractor.XPathResult("//meta[@itemprop='uploadDate']/@content")
+		if err != nil {
+			log.Println(err)
+		}
+
+		iter := content.NodeIter()
+		if iter.Next() {
+			tm, err := time.ParseInLocation("2006-01-02T15:04:05Z07:00", iter.Node().NodeValue(), time.Local)
+			if err != nil {
+				log.Println(err)
+			}
+			clog.Set("LiveStartTime", sql.NullTime{Time: tm.Local(), Valid: true})
+
+			duration, err := extractor.XPathResult("//meta[@itemprop='duration']/@content")
+			if err != nil {
+				log.Println(err)
+			}
+
+			diter := duration.NodeIter()
+			if diter.Next() {
+
+				dt, err := intimate.ParseDuration(diter.Node().NodeValue())
+				if err != nil {
+					log.Println(err)
+				}
+				endtm := tm.Add(dt)
+				clog.Set("LiveEndTime", sql.NullTime{Time: endtm.Local(), Valid: true})
+			}
+		}
+	}
+}
+
+func (oe *OpenrecExtractor) extractTags(clog intimate.ISet) {
+	var tags []string
+	matheslist := regexp.MustCompile(`<[^>]+TagButton[^>]+>([^<]{1,100})<`).FindAllStringSubmatch(oe.userLive.GetSource().Str, -1)
+	for _, m := range matheslist {
+		tags = append(tags, m[1])
+	}
+	log.Println(tags)
+	tagsBytes, err := json.Marshal(tags)
+	if err != nil {
+		log.Println(err)
+	}
+
+	clog.Set("Tags", tagsBytes)
+}

extractor/openrec_extractor/openrec_test.go

@@ -0,0 +1,96 @@
+package main
+
+import (
+	"io/ioutil"
+	"os"
+	"regexp"
+	"testing"
+	"time"
+
+	"github.com/lestrrat-go/libxml2"
+)
+
+func TestCase0(t *testing.T) {
+	f, err := os.Open("./test.html")
+	if err != nil {
+		panic(err)
+	}
+	data, err := ioutil.ReadAll(f)
+	if err != nil {
+		panic(err)
+	}
+
+	matheslist := regexp.MustCompile(`TagButton__Button[^>]+>(.{1,100})</a`).FindAllStringSubmatch(string(data), -1)
+	t.Error(matheslist)
+}
+
+func TestCase1(t *testing.T) {
+	date := "2020-07-13T18:58:24+09:00"
+
+	tm, err := time.Parse("2006-01-02T15:04:05Z07:00", date)
+	t.Error(err)
+	t.Error(time.Now())
+	t.Error(tm.Local().UTC(), tm.Local())
+
+}
+
+func TestCase2(t *testing.T) {
+	duration1 := "0:00:00"
+	duration2 := "4:56:04"
+	tm2, err := time.Parse("15:04:05", duration2)
+	tm1, err := time.Parse("15:04:05", duration1)
+
+	tm2.Sub(tm1)
+
+	t.Error(err)
+	t.Error(tm2.Sub(tm1))
+
+}
+
+func TestCase(t *testing.T) {
+	f, _ := os.Open("./test.html")
+	data, _ := ioutil.ReadAll(f)
+
+	doc, err := libxml2.ParseHTML(data)
+	if err != nil {
+		panic(err)
+	}
+	// doc.CreateElement("meta")
+	// "<META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=utf-8">"
+
+	xresult, err := doc.Find("/html/head")
+	ele, err := doc.CreateElement(`META`)
+
+	if err != nil {
+		panic(err)
+	}
+	ele.SetAttribute("charset", "utf-8")
+
+	if err != nil {
+		panic(err)
+	}
+
+	iter := xresult.NodeIter()
+	if iter.Next() {
+		n := iter.Node()
+
+		err = n.AddChild(ele)
+		// childs, err := n.ChildNodes()
+		if err != nil {
+			t.Error(err)
+		}
+		t.Error(n)
+	}
+
+	xr, err := doc.Find("//h1[ contains(@class, 'MovieTitle__Title')]")
+	if err != nil {
+		panic(nil)
+	}
+
+	t.Error(xr)
+}
+
+func TestExtractor(t *testing.T) {
+	oe := &OpenrecExtractor{}
+	oe.Execute()
+}