commit c0a81a3c5f4aca74dcac6ec6faa6ead21d453437 Author: huangsimin Date: Mon Nov 26 09:30:02 2018 +0800 yaml test success! diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f4dde5b --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +*.pyc +*.vscode +*.test diff --git a/config.go b/config.go new file mode 100644 index 0000000..8077e15 --- /dev/null +++ b/config.go @@ -0,0 +1,96 @@ +package imitate + +import ( + "io/ioutil" + "log" + "os" + "strings" + + yaml "gopkg.in/yaml.v2" +) + +// Config 任务加载的默认配置 +type Config struct { + Session int `yaml:"session"` + Mode int `yaml:"mode"` + Proxies []string `yaml:"proxies"` + Retry int `yaml:"retry"` + Priority int `yaml:"priority"` + Curl string `yaml:"curl"` + + ExecuteInterval `yaml:"execute_interval"` + ExecuteAt `yaml:"execute_at"` + + Device string `yaml:"device"` + Platform string `yaml:"platform"` + AreaCC string `yaml:"area_cc"` + Channel int `yaml:"channel"` + Media int `yaml:"media"` + SpiderID int `yaml:"spider_id"` + CatchAccountID int `yaml:"catch_account_id"` +} + +// newDefaultConfig create a default config +func newDefaultConfig() *Config { + conf := &Config{ + Session: 1, + Mode: 0, + Retry: 0, + Priority: 10000, + + ExecuteInterval: ExecuteInterval{ + TimeInterval: -1, + }, + + ExecuteAt: ExecuteAt{ + Year: -1, + Month: -1, + Day: -1, + Hour: -1, + Min: -1, + Sec: -1, + }, + + Device: "", + Platform: "", + AreaCC: "", + Channel: -1, + Media: -1, + SpiderID: -1, + CatchAccountID: -1, + } + return conf +} + +// NewConfig 加载并返回Config +func NewConfig(p string) *Config { + f, err := os.Open(p) + defer f.Close() + if err != nil { + panic(err) + } + + conf := newDefaultConfig() + + err = yaml.NewDecoder(f).Decode(conf) + if err != nil { + panic(err) + } + // spew.Dump(conf) + + if conf.Curl == "" { + log.Println("the path ", p, "curl is \"\"") + } else { + if conf.Curl[0] == '@' { + curlfile, err := os.Open(conf.Curl[1:]) + defer curlfile.Close() + if err != nil { + panic(err) + } + curldata, err := ioutil.ReadAll(curlfile) + conf.Curl = strings.Trim(string(curldata), "\r\n ") + } + } + + return conf +} diff --git a/execute_at.go b/execute_at.go new file mode 100644 index 0000000..a3c6b66 --- /dev/null +++ b/execute_at.go @@ -0,0 +1,89 @@ +package imitate + +import ( + "time" +) + +// ExecuteAt 特定的时间任务 接口源自于 IExecute +type ExecuteAt struct { + Year int `yaml:"year"` + Month int `yaml:"month"` + Day int `yaml:"day"` + Hour int `yaml:"hour"` + Min int `yaml:"min"` + Sec int `yaml:"sec"` + + TriggerTime int64 // 下次的触发时间点 + StartStatus bool // 一个值判断这个时间表是否有效 +} + +// SetStartStatus 设置执行计划是否生效 +func (ea *ExecuteAt) SetStartStatus(status bool) { + ea.StartStatus = status +} + +// GetTriggerTime 获取计划的触发时间 +func (ea *ExecuteAt) GetTriggerTime() int64 { + if ea.StartStatus { + return ea.TriggerTime + } + return -1 +} + +// TimeTo 是否到了该触发的时间 +func (ea *ExecuteAt) TimeTo() int64 { + return time.Now().Unix() - ea.TriggerTime +} + +// GetStartStatus 获取计划的触发时间是否在生效 +func (ea *ExecuteAt) GetStartStatus() bool { + return ea.StartStatus +} + +// CalculateTrigger 计算触发特定时间任务的时间点 执行后 可以通过GetTriggerTime确认触发时间 +func (ea *ExecuteAt) CalculateTrigger() int64 { + now := time.Now() + + year := ea.Year + if ea.Year <= 0 { + year = now.Year() + } + + month := time.Month(ea.Month) + if ea.Month <= 0 { + month = now.Month() + } + + day := ea.Day + if ea.Day <= 0 { + day = now.Day() + } + + hour := ea.Hour + if ea.Hour < 0 { + hour = now.Hour() + } + + min := ea.Min + if ea.Min < 0 { + min = now.Minute() + } + + sec := ea.Sec + if ea.Sec < 0 { + sec = now.Second() + } + + ea.TriggerTime = time.Date(year, time.Month(month), day, hour, min, sec, 0, time.Local).Unix() + return ea.TriggerTime +} + +// FromValues 从数值 里获取执行时间表的结构 +func (ea *ExecuteAt) FromValues(year int, month int, day int, hour int, min int, sec int) { + ea.Year = year + ea.Month = month + ea.Day = day + ea.Hour = hour + ea.Min = min + ea.Sec = sec +} diff --git a/execute_interval.go b/execute_interval.go new file mode 100644 index 0000000..09a7697 --- /dev/null +++ b/execute_interval.go @@ -0,0 +1,48 @@ +package imitate + +import ( + "time" +) + +// ExecuteInterval 时间间隔的类型 +type ExecuteInterval struct { + TimeInterval int64 `yaml:"sec"` // 时间间隔 + + TriggerTime int64 // 执行时间间隔触发时间 + StartStatus bool // 判断是否按照时间间隔执行 +} + +// SetStartStatus 设置执行计划是否生效 +func (ei *ExecuteInterval) SetStartStatus(status bool) { + ei.StartStatus = status +} + +// GetTriggerTime 获取计划的触发时间 +func (ei *ExecuteInterval) GetTriggerTime() int64 { + if ei.StartStatus { + return ei.TriggerTime + } + return -1 +} + +// TimeTo 是否到了该触发的时间 +func (ei *ExecuteInterval) TimeTo() int64 { + return time.Now().Unix() - ei.TriggerTime +} + +// GetStartStatus 获取计划的触发时间是否在生效 +func (ei *ExecuteInterval) GetStartStatus() bool { + return ei.StartStatus +} + +// CalculateTrigger 计算触发特定时间任务的时间点 +func (ei *ExecuteInterval) CalculateTrigger() int64 { + now := time.Now() + ei.TriggerTime = now.Unix() + ei.TimeInterval + return ei.TriggerTime +} + +// FromValue 生成计划表 +func (ei *ExecuteInterval) FromValue(vsleep int64) { + ei.TimeInterval = vsleep +} diff --git a/execute_plan.go b/execute_plan.go new file mode 100644 index 0000000..9b3c8e1 --- /dev/null +++ b/execute_plan.go @@ -0,0 +1,50 @@ +package imitate + +import ( + "github.com/474420502/requests" +) + +// type ExecuteRecord struct { +// IsSuccess bool +// FailCount int +// Message []string +// } + +// IExecute 计划执行的时间接口 +type IExecute interface { + SetStartStatus(status bool) // SetStart 设置执行计划是否生效 + GetStartStatus() bool // IsStart 获取计划的触发时间是否在生效 + GetTriggerTime() int64 // GetTriggerTime 获取计划的触发时间 + + TimeTo() int64 // TimeTo 是否到了该触发的时间 + + // SetSuccessStatus(status bool) // SetSuccessStatus 设置成功的状态 将记录于历史 + // History() []ExecuteRecord // History 记录一些历史, 可能会持久到数据库. 暂时不要 + CalculateTrigger() int64 // CalculateTrigger 计算触发特定时间任务的时间点 +} + +// ExecutePlan 执行时间的计划表 +type ExecutePlan struct { + ExecuteQueue []IExecute +} + +// PlanResult 执行计划表的结果 +type PlanResult struct { + Exec IExecute + Resp *requests.Response +} + +// AppendIExecute 添加执行计划任务 +func (ep *ExecutePlan) AppendIExecute(e IExecute) { + ep.ExecuteQueue = append(ep.ExecuteQueue, e) +} + +// ClearIExecute 清除执行计划任务 +func (ep *ExecutePlan) ClearIExecute() { + ep.ExecuteQueue = []IExecute{} +} + +// CountIExecute 清除执行计划任务 +func (ep *ExecutePlan) CountIExecute() int { + return len(ep.ExecuteQueue) +} diff --git a/execute_test.go b/execute_test.go new file mode 100644 index 0000000..d150377 --- /dev/null +++ b/execute_test.go @@ -0,0 +1,43 @@ +package imitate + +import ( + "log" + "regexp" + "testing" + + "github.com/davecgh/go-spew/spew" + + "474420502.top/eson/curl2info" +) + +type Person struct { + Tasks []Task +} + +type Task struct { + GURL *curl2info.CURL + Plan *ExecutePlan +} + +func TestNewYaml(t *testing.T) { + data := spew.Sdump(NewConfig("test.yaml")) + if !(regexp.MustCompile(`Device: \(string\) \(len=12\) "eson-OnePlus"`).MatchString(data) && regexp.MustCompile(`Sec: \(int\) 30`).MatchString(data)) { + t.Error(data) + } +} + +func TestExecute(t *testing.T) { + curl := `curl 'https://re.csdn.net/csdnbi' -H 'Origin: https://blog.csdn.net' -H 'Accept-Encoding: gzip, deflate, br' -H 'Accept-Language: zh' -H 'User-Agent: Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1' -H 'Content-Type: text/plain;charset=UTF-8' -H 'Accept: */*' -H 'Referer: https://blog.csdn.net/levon2018/article/details/80558108' -H 'Cookie: uuid_tt_dd=10_30744014340-1533110459408-320601; smidV2=2018082914272667751d1cac24793525732f74f62cbe9a0099f94a08e9b7de0; dc_session_id=10_1536825785870.531052; ARK_ID=JSa3a7b5f009df2bf82fa457adf0741aaaa3a7; Hm_lvt_6bcd52f51e9b3dce32bec4a3997715ac=1538100545,1538118577,1538119202,1538121038; dc_tos=pfrclv; Hm_lpvt_6bcd52f51e9b3dce32bec4a3997715ac=1538124692' -H 'Connection: keep-alive' --data-binary '[{"headers":{"component":"enterprise","datatype":"re","version":"v1"},"body":"{\"re\":\"uid=-&ref=&pid=blog&mod=popu_36&con=%2Chttps%3A%2F%2Fblog.csdn.net%2Fxixi880928%2Farticle%2Fdetails%2F78339157%2CBlogCommendFromBaidu_2&ck=-&curl=https%3A%2F%2Fblog.csdn.net%2Flevon2018%2Farticle%2Fdetails%2F80558108&session_id=10_1536825785870.531052&type=view\"}"},{"headers":{"component":"enterprise","datatype":"re","version":"v1"},"body":"{\"re\":\"uid=-&ref=&pid=blog&mod=popu_36&con=%2Chttps%3A%2F%2Fblog.csdn.net%2Fchaowanghn%2Farticle%2Fdetails%2F54601024%2CBlogCommendFromBaidu_3&ck=-&curl=https%3A%2F%2Fblog.csdn.net%2Flevon2018%2Farticle%2Fdetails%2F80558108&session_id=10_1536825785870.531052&type=view\"}"},{"headers":{"component":"enterprise","datatype":"re","version":"v1"},"body":"{\"re\":\"uid=-&ref=&pid=blog&mod=popu_36&con=%2Chttps%3A%2F%2Fblog.csdn.net%2Flanchunhui%2Farticle%2Fdetails%2F78877514%2CBlogCommendFromBaidu_4&ck=-&curl=https%3A%2F%2Fblog.csdn.net%2Flevon2018%2Farticle%2Fdetails%2F80558108&session_id=10_1536825785870.531052&type=view\"}"},{"headers":{"component":"enterprise","datatype":"re","version":"v1"},"body":"{\"re\":\"uid=-&ref=&pid=blog&mod=popu_36&con=%2Chttps%3A%2F%2Fblog.csdn.net%2Fsaw1990%2Farticle%2Fdetails%2F39287631%2CBlogCommendFromBaidu_5&ck=-&curl=https%3A%2F%2Fblog.csdn.net%2Flevon2018%2Farticle%2Fdetails%2F80558108&session_id=10_1536825785870.531052&type=view\"}"},{"headers":{"component":"enterprise","datatype":"re","version":"v1"},"body":"{\"re\":\"uid=-&ref=&pid=blog&mod=popu_36&con=%2Chttps%3A%2F%2Fblog.csdn.net%2Fu011044684%2Farticle%2Fdetails%2F45287575%2CBlogCommendFromBaidu_6&ck=-&curl=https%3A%2F%2Fblog.csdn.net%2Flevon2018%2Farticle%2Fdetails%2F80558108&session_id=10_1536825785870.531052&type=view\"}"},{"headers":{"component":"enterprise","datatype":"re","version":"v1"},"body":"{\"re\":\"uid=-&ref=&pid=blog&mod=popu_36&con=%2Chttps%3A%2F%2Fblog.csdn.net%2Fnimade511%2Farticle%2Fdetails%2F52540437%2CBlogCommendFromBaidu_7&ck=-&curl=https%3A%2F%2Fblog.csdn.net%2Flevon2018%2Farticle%2Fdetails%2F80558108&session_id=10_1536825785870.531052&type=view\"}"},{"headers":{"component":"enterprise","datatype":"re","version":"v1"},"body":"{\"re\":\"uid=-&ref=&pid=blog&mod=popu_36&con=%2Chttps%3A%2F%2Fblog.csdn.net%2Fpipisorry%2Farticle%2Fdetails%2F47948065%2CBlogCommendFromBaidu_8&ck=-&curl=https%3A%2F%2Fblog.csdn.net%2Flevon2018%2Farticle%2Fdetails%2F80558108&session_id=10_1536825785870.531052&type=view\"}"},{"headers":{"component":"enterprise","datatype":"re","version":"v1"},"body":"{\"re\":\"uid=-&ref=&pid=blog&mod=popu_36&con=%2Chttps%3A%2F%2Fblog.csdn.net%2Fyudiyanwang%2Farticle%2Fdetails%2F71843633%2CBlogCommendFromBaidu_9&ck=-&curl=https%3A%2F%2Fblog.csdn.net%2Flevon2018%2Farticle%2Fdetails%2F80558108&session_id=10_1536825785870.531052&type=view\"}"}]'` + + u, err := curl2info.ParseRawCURL(curl) + if err != nil { + panic(err) + } + + resp, err := u.CreateWorkflow(nil).Execute() + if err != nil { + t.Error("TestExecute") + } else { + log.Println(resp.Content()) + } +} diff --git a/test.curl b/test.curl new file mode 100644 index 0000000..c945519 --- /dev/null +++ b/test.curl @@ -0,0 +1 @@ +curl 'http://is.snssdk.com/2/article/information/v24/?latitude=22.831367&longitude=113.511515&group_id=6565653745026204168&item_id=6565653745026204168&aggr_type=1&context=1&from_category=news_game&article_page=0&iid=34903754482&device_id=41148471494&ac=wifi&channel=oppo-cpa&aid=13&app_name=news_article&version_code=676&version_name=6.7.6&device_platform=android&ab_version=304489%2C261579%2C373245%2C360501%2C374617%2C366851%2C356335%2C345191%2C271178%2C357704%2C326524%2C326532%2C292723%2C366036%2C323233%2C371779%2C346557%2C351090%2C319958%2C372620%2C362184%2C214069%2C31643%2C333971%2C366873%2C374962%2C372618%2C280449%2C281298%2C366489%2C325619%2C373770%2C357402%2C361073%2C362402%2C290191%2C370014%2C353484%2C375739%2C373725%2C295827%2C353305%2C375426%2C374426%2C239095%2C360541%2C344347%2C170988%2C371590%2C368831%2C368827%2C368775%2C374117%2C365053%2C374232%2C368303%2C375692%2C330632%2C297059%2C374250%2C276206%2C286212%2C350193%2C365036%2C373741%2C374405%2C373368%2C370846%2C364453%2C375713%2C369501%2C369165%2C368839%2C375433%2C373123%2C371555%2C371963%2C374142%2C372907&ab_client=a1%2Cc4%2Ce1%2Cf1%2Cg2%2Cf7&ab_group=94567%2C102754%2C181430&ab_feature=94567%2C102754&abflag=3&ssmix=a&device_type=ONEPLUS+A3010&device_brand=OnePlus&language=zh&os_api=26&os_version=8.0.0&uuid=864854034514328&openudid=9b35a4035eecee2c&manifest_version_code=676&resolution=1080*1920&dpi=420&update_version_code=67610&_rticket=1528706910264&plugin=10603&pos=5r_-9Onkv6e_eCQieCoDeCUfv7G_8fLz-vTp6Pn4v6esrK6zqKysqKyosb_x_On06ej5-L-nr6-zpa6srquqsb_88Pzt3vTp5L-nv3gkIngqA3glH7-xv_zw_O3R8vP69Ono-fi_p6ysrrOupauqqaSxv_zw_O3R_On06ej5-L-nr66zrairpKqv4A%3D%3D&fp=HrT_FlD_PMcIFlD5FSU1FYmeFrxO&rom_version=26&ts=1528706911&as=a265e371dff53b57de5999&mas=0073e8ef3f9a8b842da0ead7d35c0597ea2ee0ccce5e5d5db5' -H 'Accept-Encoding: gzip' -H 'X-SS-REQ-TICKET: 1528706910267' -H 'User-Agent: Dalvik/2.1.0 (Linux; U; Android 8.0.0; ONEPLUS A3010 Build/OPR1.170623.032) NewsArticle/6.7.6 okhttp/3.10.0.1' -H 'Cookie: odin_tt=210899a257b5fe787a3465e2220fb94d91d5ad34c77dee3560f93fccc82dd738cccb301770f633530fdd6ceea955983d; UM_distinctid=163ace3b0050-08fccf530af621-f1c0e26-49a10-163ace3b0093e8; CNZZDATA1271720685=1435124261-1527612007-%7C1527612007; CNZZDATA1264530760=119491224-1527609979-%7C1527612115; JSESSIONID=67814B7DDE08D5A9F3B3D684220CF3FB; alert_coverage=6; qh[360]=1; install_id=34903754482; ttreq=1$b7221ef01bd5ed7c030f5db45e959686c9ddd0d2' -H 'Host: is.snssdk.com' -H 'Connection: Keep-Alive' \ No newline at end of file diff --git a/test.yaml b/test.yaml new file mode 100644 index 0000000..6805794 --- /dev/null +++ b/test.yaml @@ -0,0 +1,29 @@ +session : 1 +mode : 0 +proxies : ["socks5://10.10.10.1:8080", "socks5://10.10.10.1:8082", "socks5://10.10.10.1:8083", "socks5://10.10.10.1:8085", "socks5://10.10.10.1:8087", "socks5://10.10.10.1:8088", "socks5://10.10.10.1:8089", "socks5://10.10.10.1:8090"] +retry : 0 +priority : 10000 + +curl : "@test.curl" +# next_do : "doothers" + + + +device : "eson-OnePlus" +platform : "Android" +area_cc : 4401 +channel : 105 +media : 55 +spider_id : 73 +catch_account_id : 123 + +execute_interval: # 时间间隔执行 + sec: 2 + +execute_at : # (-1, -1, -1, -1, -1, 30) 时间 每30秒执行 min 设置20的时候 = 每小时的20分钟30秒执行 + year : -1 + month : -1 + day : -1 + hour : -1 + min : -1 + sec : 30 \ No newline at end of file