From 0f8201566c3e0078a9b9528c0be6f7480aba53b4 Mon Sep 17 00:00:00 2001 From: huangsimin Date: Tue, 27 Nov 2018 17:46:29 +0800 Subject: [PATCH] TODO: CallBack! Test Reload --- base.go | 1 + config.go | 2 - execute_at.go | 3 +- execute_interval.go | 3 +- execute_plan.go | 4 +- execute_test.go | 78 +--------------------------- structure.go | 51 +++++++++---------- task.go | 120 ++++++++++++++++++++++++++++++++++++++++++++ task_test.go | 42 ++++++++++++++++ test.yaml | 1 + 10 files changed, 194 insertions(+), 111 deletions(-) create mode 100644 base.go create mode 100644 task.go create mode 100644 task_test.go diff --git a/base.go b/base.go new file mode 100644 index 0000000..cab97bd --- /dev/null +++ b/base.go @@ -0,0 +1 @@ +package imitate diff --git a/config.go b/config.go index bf17d1d..749220d 100644 --- a/config.go +++ b/config.go @@ -7,7 +7,6 @@ import ( "reflect" "strings" - "github.com/davecgh/go-spew/spew" yaml "gopkg.in/yaml.v2" ) @@ -116,7 +115,6 @@ func NewConfig(p string) *Config { if err != nil { panic(err) } - spew.Dump(conf) return conf } diff --git a/execute_at.go b/execute_at.go index a3c6b66..ab2d520 100644 --- a/execute_at.go +++ b/execute_at.go @@ -41,7 +41,7 @@ func (ea *ExecuteAt) GetStartStatus() bool { } // CalculateTrigger 计算触发特定时间任务的时间点 执行后 可以通过GetTriggerTime确认触发时间 -func (ea *ExecuteAt) CalculateTrigger() int64 { +func (ea *ExecuteAt) CalculateTrigger() { now := time.Now() year := ea.Year @@ -75,7 +75,6 @@ func (ea *ExecuteAt) CalculateTrigger() int64 { } ea.TriggerTime = time.Date(year, time.Month(month), day, hour, min, sec, 0, time.Local).Unix() - return ea.TriggerTime } // FromValues 从数值 里获取执行时间表的结构 diff --git a/execute_interval.go b/execute_interval.go index 09a7697..8787f65 100644 --- a/execute_interval.go +++ b/execute_interval.go @@ -36,10 +36,9 @@ func (ei *ExecuteInterval) GetStartStatus() bool { } // CalculateTrigger 计算触发特定时间任务的时间点 -func (ei *ExecuteInterval) CalculateTrigger() int64 { +func (ei *ExecuteInterval) CalculateTrigger() { now := time.Now() ei.TriggerTime = now.Unix() + ei.TimeInterval - return ei.TriggerTime } // FromValue 生成计划表 diff --git a/execute_plan.go b/execute_plan.go index e3242ab..a69a4e5 100644 --- a/execute_plan.go +++ b/execute_plan.go @@ -16,11 +16,11 @@ type IExecute interface { // SetSuccessStatus(status bool) // SetSuccessStatus 设置成功的状态 将记录于历史 // History() []ExecuteRecord // History 记录一些历史, 可能会持久到数据库. 暂时不要 - CalculateTrigger() int64 // CalculateTrigger 计算触发特定时间任务的时间点 + CalculateTrigger() // CalculateTrigger 计算触发特定时间任务的时间点 } // ExecutePlan 执行时间的计划表 -type ExecutePlan CircularLinked +// type ExecutePlan CircularLinked // // PlanResult 执行计划表的结果 // type PlanResult struct { diff --git a/execute_test.go b/execute_test.go index afe3d75..ec9fa02 100644 --- a/execute_test.go +++ b/execute_test.go @@ -1,71 +1,12 @@ package imitate import ( - "log" "regexp" "testing" "github.com/davecgh/go-spew/spew" - - "474420502.top/eson/curl2info" ) -type Person struct { - Tasks *CircularLinked - Conf *Config -} - -type Task struct { - Curl *curl2info.CURL - Plan *CircularLinked - Proxies *CircularLinked -} - -func NewPerson(conf string, curlinfo string) *Person { - person := &Person{} - person.Tasks = NewCircularLinked() - person.Conf = NewConfig(conf) - - if person.Conf.Mode == 0 { - for _, scurl := range person.Conf.Curls { - curl, err := curl2info.ParseRawCURL(scurl) - if err != nil { - panic(err) - } - at := person.Conf.ExecuteAt - interval := person.Conf.ExecuteInterval - task := NewTask(curl, &at, &interval) - switch person.Conf.Mode { - case 0: - task.Proxies.Append(person.Conf.Proxies) - person.Tasks.Append(task) - case 1: - for _, proxy := range person.Conf.Proxies { - curl := &*task.Curl - curl.Auth = &*task.Curl.Auth - curl.Body = &*task.Curl.Body - curl.ParsedURL = &*task.Curl.ParsedURL - ptask := NewTask(curl) - task.Plan.Clone() - } - - } - } - } - - return person -} - -func NewTask(Curl *curl2info.CURL, Plans ...IExecute) *Task { - task := &Task{} - // task.Conf = NewConfig(conf) - task.Curl = Curl - task.Plan = NewCircularLinked(Plans) - task.Proxies = NewCircularLinked() - - return task -} - func TestNewYaml(t *testing.T) { data := spew.Sdump(NewConfig("test.yaml")) if !(regexp.MustCompile(`Device: \(string\) \(len=12\) "eson-OnePlus"`).MatchString(data) && regexp.MustCompile(`Sec: \(int\) 30`).MatchString(data) && regexp.MustCompile(`http://is.snssdk.com/2/article/information/v24/\?`).MatchString(data)) { @@ -73,22 +14,5 @@ func TestNewYaml(t *testing.T) { } } -func TestExecute(t *testing.T) { - curl := `curl 'https://re.csdn.net/csdnbi' -H 'Origin: https://blog.csdn.net' -H 'Accept-Encoding: gzip, deflate, br' -H 'Accept-Language: zh' -H 'User-Agent: Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1' -H 'Content-Type: text/plain;charset=UTF-8' -H 'Accept: */*' -H 'Referer: https://blog.csdn.net/levon2018/article/details/80558108' -H 'Cookie: uuid_tt_dd=10_30744014340-1533110459408-320601; smidV2=2018082914272667751d1cac24793525732f74f62cbe9a0099f94a08e9b7de0; dc_session_id=10_1536825785870.531052; ARK_ID=JSa3a7b5f009df2bf82fa457adf0741aaaa3a7; Hm_lvt_6bcd52f51e9b3dce32bec4a3997715ac=1538100545,1538118577,1538119202,1538121038; dc_tos=pfrclv; Hm_lpvt_6bcd52f51e9b3dce32bec4a3997715ac=1538124692' -H 'Connection: keep-alive' --data-binary '[{"headers":{"component":"enterprise","datatype":"re","version":"v1"},"body":"{\"re\":\"uid=-&ref=&pid=blog&mod=popu_36&con=%2Chttps%3A%2F%2Fblog.csdn.net%2Fxixi880928%2Farticle%2Fdetails%2F78339157%2CBlogCommendFromBaidu_2&ck=-&curl=https%3A%2F%2Fblog.csdn.net%2Flevon2018%2Farticle%2Fdetails%2F80558108&session_id=10_1536825785870.531052&type=view\"}"},{"headers":{"component":"enterprise","datatype":"re","version":"v1"},"body":"{\"re\":\"uid=-&ref=&pid=blog&mod=popu_36&con=%2Chttps%3A%2F%2Fblog.csdn.net%2Fchaowanghn%2Farticle%2Fdetails%2F54601024%2CBlogCommendFromBaidu_3&ck=-&curl=https%3A%2F%2Fblog.csdn.net%2Flevon2018%2Farticle%2Fdetails%2F80558108&session_id=10_1536825785870.531052&type=view\"}"},{"headers":{"component":"enterprise","datatype":"re","version":"v1"},"body":"{\"re\":\"uid=-&ref=&pid=blog&mod=popu_36&con=%2Chttps%3A%2F%2Fblog.csdn.net%2Flanchunhui%2Farticle%2Fdetails%2F78877514%2CBlogCommendFromBaidu_4&ck=-&curl=https%3A%2F%2Fblog.csdn.net%2Flevon2018%2Farticle%2Fdetails%2F80558108&session_id=10_1536825785870.531052&type=view\"}"},{"headers":{"component":"enterprise","datatype":"re","version":"v1"},"body":"{\"re\":\"uid=-&ref=&pid=blog&mod=popu_36&con=%2Chttps%3A%2F%2Fblog.csdn.net%2Fsaw1990%2Farticle%2Fdetails%2F39287631%2CBlogCommendFromBaidu_5&ck=-&curl=https%3A%2F%2Fblog.csdn.net%2Flevon2018%2Farticle%2Fdetails%2F80558108&session_id=10_1536825785870.531052&type=view\"}"},{"headers":{"component":"enterprise","datatype":"re","version":"v1"},"body":"{\"re\":\"uid=-&ref=&pid=blog&mod=popu_36&con=%2Chttps%3A%2F%2Fblog.csdn.net%2Fu011044684%2Farticle%2Fdetails%2F45287575%2CBlogCommendFromBaidu_6&ck=-&curl=https%3A%2F%2Fblog.csdn.net%2Flevon2018%2Farticle%2Fdetails%2F80558108&session_id=10_1536825785870.531052&type=view\"}"},{"headers":{"component":"enterprise","datatype":"re","version":"v1"},"body":"{\"re\":\"uid=-&ref=&pid=blog&mod=popu_36&con=%2Chttps%3A%2F%2Fblog.csdn.net%2Fnimade511%2Farticle%2Fdetails%2F52540437%2CBlogCommendFromBaidu_7&ck=-&curl=https%3A%2F%2Fblog.csdn.net%2Flevon2018%2Farticle%2Fdetails%2F80558108&session_id=10_1536825785870.531052&type=view\"}"},{"headers":{"component":"enterprise","datatype":"re","version":"v1"},"body":"{\"re\":\"uid=-&ref=&pid=blog&mod=popu_36&con=%2Chttps%3A%2F%2Fblog.csdn.net%2Fpipisorry%2Farticle%2Fdetails%2F47948065%2CBlogCommendFromBaidu_8&ck=-&curl=https%3A%2F%2Fblog.csdn.net%2Flevon2018%2Farticle%2Fdetails%2F80558108&session_id=10_1536825785870.531052&type=view\"}"},{"headers":{"component":"enterprise","datatype":"re","version":"v1"},"body":"{\"re\":\"uid=-&ref=&pid=blog&mod=popu_36&con=%2Chttps%3A%2F%2Fblog.csdn.net%2Fyudiyanwang%2Farticle%2Fdetails%2F71843633%2CBlogCommendFromBaidu_9&ck=-&curl=https%3A%2F%2Fblog.csdn.net%2Flevon2018%2Farticle%2Fdetails%2F80558108&session_id=10_1536825785870.531052&type=view\"}"}]'` - - u, err := curl2info.ParseRawCURL(curl) - if err != nil { - panic(err) - } - - resp, err := u.CreateWorkflow(nil).Execute() - if err != nil { - t.Error("TestExecute") - } else { - log.Println(resp.Content()) - } -} - -func TestExecutePlan(t *testing.T) { - +func TestCase(t *testing.T) { } diff --git a/structure.go b/structure.go index 35d8698..4b2dadc 100644 --- a/structure.go +++ b/structure.go @@ -1,8 +1,6 @@ package imitate import ( - "bytes" - "encoding/gob" "log" "strings" @@ -16,6 +14,16 @@ type Node struct { next *Node } +// GetValue 获取到Node的值 +func (node *Node) GetValue() interface{} { + return node.value +} + +// SetValue 获取到Node的值 +func (node *Node) SetValue(value interface{}) { + node.value = value +} + // CircularLinked 循环链表 type CircularLinked struct { cursor *Node @@ -71,6 +79,19 @@ func (list *CircularLinked) CursorToTail() *Node { return list.cursor } +// GetLoopValues 获取从头到尾的值 +func (list *CircularLinked) GetLoopValues() []*Node { + var result []*Node + + if list.head != nil { + result = append(result, list.head) + for cur := list.head.next; cur != list.head; cur = cur.next { + result = append(result, cur) + } + } + return result +} + // Append a value (one or more) at the end of the list (same as Append()) func (list *CircularLinked) Append(values ...interface{}) { for _, value := range values { @@ -176,7 +197,7 @@ func (list *CircularLinked) LookCursor() string { content = strings.TrimRight(content, ", ") showlen := len(content) if showlen >= 64 { - showlen = 32 + showlen = 64 } content += "]" + content[0:showlen] + " ..." return content @@ -199,38 +220,16 @@ func (list *CircularLinked) Size() uint64 { return list.size } -// Clone 复制一个完整的链表, 只是value引用, 如果是指针他们(链表)之间, 操作value会可能产生影响 -func (list *CircularLinked) Clone() *CircularLinked { - cl := NewCircularLinked() - for i := uint64(0); i < list.size; i++ { - var buf bytes.Buffer // Stand-in for a network connection - enc := gob.NewEncoder(&buf) // Will write to network. - dec := gob.NewDecoder(&buf) // Will read from network. - // Encode (send) the value. - err := enc.Encode(list.cursor.value) - if err != nil { - log.Fatal("encode error:", err) - } - var clonevalue interface{} - err = dec.Decode(clonevalue) - if err != nil { - panic(err) - } - cl.Append(clonevalue) - } - return cl -} - func (list *CircularLinked) errorNotInList(node *Node) { log.Println("the node value ", spew.Sprint(node), " is not in list") } +// cutAndSplice 不考虑边界情况 上层使用的是否判断 func (list *CircularLinked) cutAndSplice(node *Node) (prev, next *Node) { prev = node.prev next = node.next prev.next = next - next.prev = prev node.prev = nil diff --git a/task.go b/task.go new file mode 100644 index 0000000..57eaf92 --- /dev/null +++ b/task.go @@ -0,0 +1,120 @@ +package imitate + +import ( + "474420502.top/eson/curl2info" + "474420502.top/eson/requests" +) + +// Person 以人为单位 +type Person struct { + Tasks *CircularLinked + Conf *Config +} + +// NewPerson 创建一个人实例 +func NewPerson(conf string) *Person { + person := &Person{} + person.Tasks = NewCircularLinked() + person.Conf = NewConfig(conf) + + for _, scurl := range person.Conf.Curls { + curl, err := curl2info.ParseRawCURL(scurl) + if err != nil { + panic(err) + } + + at := *&person.Conf.ExecuteAt + interval := *&person.Conf.ExecuteInterval + + task := NewTask(curl, &at, &interval) + switch person.Conf.Mode { + case 0: + task.Proxies.Append(person.Conf.Proxies) + person.Tasks.Append(task) + case 1: + for _, proxy := range person.Conf.Proxies { + ncurl, err := curl2info.ParseRawCURL(scurl) + if err != nil { + panic(err) + } + + ptask := NewTask(ncurl) + for _, exec := range task.Plan.GetLoopValues() { + switch v := exec.GetValue().(type) { + case *ExecuteAt: + clone := &*v + ptask.Plan.Append(clone) + case *ExecuteInterval: + clone := &*v + ptask.Plan.Append(clone) + } + } + + ptask.Proxies.Append(proxy) + person.Tasks.Append(ptask) + } + } + } + + return person +} + +// Task 任务 +type Task struct { + Curl *curl2info.CURL + Workflow *requests.Workflow + Plan *CircularLinked + Proxies *CircularLinked +} + +// PlanResult 执行计划后返回的结果 +type PlanResult struct { + OK bool + Error error + CallBack string + Response *requests.Response +} + +// NewTask 生成一个新任务 +func NewTask(Curl *curl2info.CURL, Plans ...IExecute) *Task { + + task := &Task{} + // task.Conf = NewConfig(conf) + task.Curl = Curl + task.Plan = NewCircularLinked() + task.Proxies = NewCircularLinked() + + if len(Plans) != 0 { + for _, plan := range Plans { + task.Plan.Append(plan) + } + } + + return task +} + +// ExecuteOnPlan 按照计划执行任务并返回结果 +func (task *Task) ExecuteOnPlan() []*PlanResult { + var PRList []*PlanResult + for _, exec := range task.Plan.GetLoopValues() { + pr := &PlanResult{Error: nil, OK: false} + iexec := exec.GetValue().(IExecute) + if iexec.TimeTo() >= 0 { + resp, err := task.Execute() + if err != nil { + pr.Error = err + } else { + pr.OK = true + pr.Response = resp + } + iexec.CalculateTrigger() + } + PRList = append(PRList, pr) + } + return PRList +} + +// Execute 根据curl信息执行, TODO: 超时设置 +func (task *Task) Execute() (*requests.Response, error) { + return task.Curl.CreateWorkflow(nil).Execute() +} diff --git a/task_test.go b/task_test.go new file mode 100644 index 0000000..d76ad3e --- /dev/null +++ b/task_test.go @@ -0,0 +1,42 @@ +package imitate + +import ( + "log" + "testing" + "time" + + "474420502.top/eson/curl2info" +) + +func TestExecute(t *testing.T) { + curl := `curl 'https://appgrowing.cn/' -H 'authority: appgrowing.cn' -H 'cache-control: max-age=0' -H 'upgrade-insecure-requests: 1' -H 'user-agent: Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1' -H 'accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8' -H 'accept-encoding: gzip, deflate, br' -H 'accept-language: zh' -H 'cookie: _ga=GA1.2.1371058419.1533104518; _gid=GA1.2.896241740.1543307916; _gat_gtag_UA_4002880_19=1' -H 'if-none-match: W/"5bf7a0a9-ca6"' -H 'if-modified-since: Fri, 23 Nov 2018 06:39:37 GMT' --compressed` + + u, err := curl2info.ParseRawCURL(curl) + if err != nil { + panic(err) + } + + resp, err := u.CreateWorkflow(nil).Execute() + if err != nil { + t.Error("TestExecute") + } else { + log.Println(resp.Content()) + } +} + +func TestExecutePlan(t *testing.T) { + person := NewPerson("test.yaml") + + time.Sleep(time.Second * 2) + + for _, task := range person.Tasks.GetLoopValues() { + for _, pr := range task.GetValue().(*Task).ExecuteOnPlan() { + if pr.OK { + log.Println(pr.Response.Content()[0:100]) + } else { + t.Error(pr.Error) + } + } + } + +} diff --git a/test.yaml b/test.yaml index c871766..b0bfc4e 100644 --- a/test.yaml +++ b/test.yaml @@ -2,6 +2,7 @@ session : 1 mode : 0 proxies : ["socks5://10.10.10.1:8080", "socks5://10.10.10.1:8082", "socks5://10.10.10.1:8083", "socks5://10.10.10.1:8085", "socks5://10.10.10.1:8087", "socks5://10.10.10.1:8088", "socks5://10.10.10.1:8089", "socks5://10.10.10.1:8090"] retry : 0 +timeout: 12 priority : 10000 # curls: "@test.curl"