intimate/extractor/openrec/openrec_test.go

package main

import (
	"database/sql"
	"intimate"
	"io/ioutil"
	"log"
	"os"
	"regexp"
	"testing"
	"time"

	"github.com/lestrrat-go/libxml2"
	"github.com/tidwall/gjson"
)

func TestCase0(t *testing.T) {
	f, err := os.Open("./test.html")
	if err != nil {
		panic(err)
	}
	data, err := ioutil.ReadAll(f)
	if err != nil {
		panic(err)
	}

	matheslist := regexp.MustCompile(`TagButton__Button[^>]+>(.{1,100})</a`).FindAllStringSubmatch(string(data), -1)
	t.Error(matheslist)
}

func TestCase1(t *testing.T) {
	date := "2020-07-13T18:58:24+09:00"

	tm, err := time.Parse("2006-01-02T15:04:05Z07:00", date)
	t.Error(err)
	t.Error(time.Now())
	t.Error(tm.Local().UTC(), tm.Local())

}

func TestCase2(t *testing.T) {
	duration1 := "0:00:00"
	duration2 := "4:56:04"
	tm2, err := time.Parse("15:04:05", duration2)
	tm1, err := time.Parse("15:04:05", duration1)

	tm2.Sub(tm1)

	t.Error(err)
	t.Error(tm2.Sub(tm1))

}

func TestCase(t *testing.T) {
	f, _ := os.Open("./test.html")
	data, _ := ioutil.ReadAll(f)

	doc, err := libxml2.ParseHTML(data)
	if err != nil {
		panic(err)
	}
	// doc.CreateElement("meta")
	// "<META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=utf-8">"

	xresult, err := doc.Find("/html/head")
	ele, err := doc.CreateElement(`META`)

	if err != nil {
		panic(err)
	}
	ele.SetAttribute("charset", "utf-8")

	if err != nil {
		panic(err)
	}

	iter := xresult.NodeIter()
	if iter.Next() {
		n := iter.Node()

		err = n.AddChild(ele)
		// childs, err := n.ChildNodes()
		if err != nil {
			t.Error(err)
		}
		t.Error(n)
	}

	xr, err := doc.Find("//h1[ contains(@class, 'MovieTitle__Title')]")
	if err != nil {
		panic(nil)
	}

	t.Error(xr)
}

func TestExtractor(t *testing.T) {
	collect := intimate.NewExtractorStore()
	store := intimate.NewSourceStore("source_openrec")

	for {
		source, err := store.Pop(string(intimate.TTOpenrecRanking), 100)
		if err != nil {
			log.Println(err)
			return
		}

		anchorId := source.GetSource().String

		ai := &intimate.AnchorInfo{}
		ai.SetAnchorId(anchorId)
		ai.SetPlatform(string(intimate.Popenrec))

		sdata := source.GetExt().([]byte)

		if gjson.ValidBytes(sdata) {
			result := gjson.ParseBytes(sdata)
			datamap := result.Map()

			oe := &OpenrecExtractor{}
			oe.user = intimate.NewExtractorSource(datamap["user"])
			oe.user.CreateExtractor()

			oe.userLive = intimate.NewExtractorSource(datamap["user_live"])
			oe.userLive.CreateExtractor()

			oe.supporters = intimate.NewExtractorSource(datamap["supporters"])

			clog := &intimate.CollectLog{}

			oe.extractFollowers(clog)
			oe.extractAnchorName(ai)
			oe.extractViewsAndLiveStreaming(clog)
			oe.extractGiversAndGratuity(clog)
			oe.extractLive(clog)
			oe.extractTags(clog)

			ai.Set("UpdateTime", source.GetUpdateTime())

			LiveUrl := "https://www.openrec.tv/live/" + anchorId
			ai.Set("LiveUrl", sql.NullString{String: LiveUrl, Valid: true})

			Uid, err := collect.InsertAnchorInfo(ai)
			if err != nil {
				t.Error(err)
				return
			}

			clog.Set("Uid", Uid)
			clog.Set("Platform", string(intimate.Popenrec))
			clog.Set("AnchorId", anchorId)
			clog.Set("UpdateTime", source.GetUpdateTime())

			collect.InsertCollectLog(clog)
		} else {
			t.Error("data is not json:\n", string(sdata))
		}
	}

}