t1.0.0

article/article.go

@@ -0,0 +1,64 @@
+package main
+
+import (
+	"log"
+	"regexp"
+
+	"github.com/tidwall/gjson"
+
+	crontab "474420502.top/eson/crontabex"
+	"474420502.top/eson/imitater"
+	"474420502.top/test/logdb"
+)
+
+func main() {
+	imitater.Register("article", &TaskEx{})
+	person := imitater.NewPerson()
+	person.Config("task.yaml")
+	person.Execute()
+}
+
+// TaskEx 任务相关类
+type TaskEx struct {
+	imitater.Task
+	db *logdb.LogDB
+}
+
+// Init 初始化函数
+func (te *TaskEx) Init() {
+	te.db = logdb.New("../logdb.yaml")
+}
+
+// Execute 执行过程的方法
+func (te *TaskEx) Execute(glocal map[string]interface{}) imitater.ITask {
+
+	resp, err := te.Request()
+	if err != nil {
+		log.Println(err)
+		return te
+	}
+
+	var adDataList []string
+	rule := regexp.MustCompile(`\\"label\\"\:\\"广告\\"|\\"label_style\\"\:3`)
+	if rule.MatchString(resp.Content()) {
+		if gjson.Valid(resp.Content()) {
+			P := gjson.Parse(resp.Content())
+			gADDate := P.Get(`data.ordered_info.#.ad_data`)
+			if gADDate.Exists() {
+				for _, result := range gADDate.Array() {
+					adDataList = append(adDataList, result.String())
+				}
+			}
+		} else {
+			log.Println("be careful:", resp.Content())
+		}
+	}
+
+	if imitater.ADDataSave(te, te.db, adDataList) {
+		te.GetCrontab().SetStatus(crontab.SExecuteOK, true)
+	} else {
+		te.GetCrontab().SetStatus(crontab.SExecuteOK, false)
+	}
+
+	return te
+}

article/article_test.go

@@ -0,0 +1,32 @@
+package main
+
+import (
+	"io/ioutil"
+	"log"
+	"regexp"
+	"testing"
+
+	"github.com/tidwall/gjson"
+)
+
+func TestTaskEx_Execute(t *testing.T) {
+	rule := regexp.MustCompile(`\\"label\\"\:\\"广告\\"|\\"label_style\\"\:3`)
+	data, err := ioutil.ReadFile("../s.json")
+	if err != nil {
+		panic(err)
+	}
+	log.Println(rule.MatchString(string(data)))
+	log.Println(gjson.Valid(string(data)))
+
+	P := gjson.Parse(string(data))
+	gADDate := P.Get(`data.ordered_info.#.ad_data`)
+	log.Println(gADDate.Exists(), gADDate.IsArray())
+
+	if !(gADDate.Exists() && gADDate.IsArray()) {
+		t.Error("path is error")
+	}
+
+	for _, result := range gADDate.Array() {
+		log.Println(gjson.Valid(result.Raw))
+	}
+}

article/task.curl

@@ -0,0 +1 @@
+curl --name "tt-article" --task "article" 'http://is.snssdk.com/2/article/information/v24/?latitude=22.831367&longitude=113.511515&group_id=6565653745026204168&item_id=6565653745026204168&aggr_type=1&context=1&from_category=news_game&article_page=0&iid=34903754482&device_id=41148471494&ac=wifi&channel=oppo-cpa&aid=13&app_name=news_article&version_code=676&version_name=6.7.6&device_platform=android&ab_version=304489%2C261579%2C373245%2C360501%2C374617%2C366851%2C356335%2C345191%2C271178%2C357704%2C326524%2C326532%2C292723%2C366036%2C323233%2C371779%2C346557%2C351090%2C319958%2C372620%2C362184%2C214069%2C31643%2C333971%2C366873%2C374962%2C372618%2C280449%2C281298%2C366489%2C325619%2C373770%2C357402%2C361073%2C362402%2C290191%2C370014%2C353484%2C375739%2C373725%2C295827%2C353305%2C375426%2C374426%2C239095%2C360541%2C344347%2C170988%2C371590%2C368831%2C368827%2C368775%2C374117%2C365053%2C374232%2C368303%2C375692%2C330632%2C297059%2C374250%2C276206%2C286212%2C350193%2C365036%2C373741%2C374405%2C373368%2C370846%2C364453%2C375713%2C369501%2C369165%2C368839%2C375433%2C373123%2C371555%2C371963%2C374142%2C372907&ab_client=a1%2Cc4%2Ce1%2Cf1%2Cg2%2Cf7&ab_group=94567%2C102754%2C181430&ab_feature=94567%2C102754&abflag=3&ssmix=a&device_type=ONEPLUS+A3010&device_brand=OnePlus&language=zh&os_api=26&os_version=8.0.0&uuid=864854034514328&openudid=9b35a4035eecee2c&manifest_version_code=676&resolution=1080*1920&dpi=420&update_version_code=67610&_rticket=1528706910264&plugin=10603&pos=5r_-9Onkv6e_eCQieCoDeCUfv7G_8fLz-vTp6Pn4v6esrK6zqKysqKyosb_x_On06ej5-L-nr6-zpa6srquqsb_88Pzt3vTp5L-nv3gkIngqA3glH7-xv_zw_O3R8vP69Ono-fi_p6ysrrOupauqqaSxv_zw_O3R_On06ej5-L-nr66zrairpKqv4A%3D%3D&fp=HrT_FlD_PMcIFlD5FSU1FYmeFrxO&rom_version=26&ts=1528706911&as=a265e371dff53b57de5999&mas=0073e8ef3f9a8b842da0ead7d35c0597ea2ee0ccce5e5d5db5' --task toutiao -H 'Accept-Encoding: gzip' -H 'X-SS-REQ-TICKET: 1528706910267' -H 'User-Agent: Dalvik/2.1.0 (Linux; U; Android 8.0.0; ONEPLUS A3010 Build/OPR1.170623.032) NewsArticle/6.7.6 okhttp/3.10.0.1' -H 'Cookie: odin_tt=210899a257b5fe787a3465e2220fb94d91d5ad34c77dee3560f93fccc82dd738cccb301770f633530fdd6ceea955983d; UM_distinctid=163ace3b0050-08fccf530af621-f1c0e26-49a10-163ace3b0093e8; CNZZDATA1271720685=1435124261-1527612007-%7C1527612007; CNZZDATA1264530760=119491224-1527609979-%7C1527612115; JSESSIONID=67814B7DDE08D5A9F3B3D684220CF3FB; alert_coverage=6; qh[360]=1; install_id=34903754482; ttreq=1$b7221ef01bd5ed7c030f5db45e959686c9ddd0d2' -H 'Host: is.snssdk.com'

article/task.yaml

@@ -0,0 +1,19 @@
+mode : 0
+# proxies : "socks5://10.10.10.1:8080" // 支持, 列表 与 单项字符串
+proxies : ["socks5://10.10.10.1:8080", "socks5://10.10.10.1:8082", "socks5://10.10.10.1:8083", "socks5://10.10.10.1:8085", "socks5://10.10.10.1:8087", "socks5://10.10.10.1:8088", "socks5://10.10.10.1:8091"]
+retry : 0
+timeout: 12
+priority : 10000
+
+curls : "@task.curl" # 支持, 列表 与 单项字符串
+
+task: "article"
+device : "eson-OnePlus"
+platform : "Android"
+# area_cc : 4401
+channel : 105
+media : 55
+spider_id : 10001001
+catch_account_id : 1001001
+
+crontab: "f8>=240|f5>=120|f=60|s=5|s10>=10|s20>=5"