From d08f6c8e68a54a7deb0ae5c73691f69e2268ad44 Mon Sep 17 00:00:00 2001 From: huangsimin Date: Fri, 7 Dec 2018 12:18:29 +0800 Subject: [PATCH] =?UTF-8?q?test=20success!=20and=20TODO:=20=E6=B7=BB?= =?UTF-8?q?=E5=8A=A0=E5=85=A8=E5=B1=80Task=E6=8C=87=E5=AE=9A=E5=8F=82?= =?UTF-8?q?=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config.go | 4 +- person.go | 127 ++++++++++++++++++++++++++++++++++++++ task.go | 170 +++++++++++++-------------------------------------- task_test.go | 3 - test.yaml | 1 - 5 files changed, 170 insertions(+), 135 deletions(-) create mode 100644 person.go diff --git a/config.go b/config.go index a2c9284..cdcbcf1 100644 --- a/config.go +++ b/config.go @@ -89,7 +89,7 @@ func (proxies *YamlProxies) MarshalYAML() (interface{}, error) { // Config 任务加载的默认配置 type Config struct { - Session int `yaml:"session"` + // Session int `yaml:"session"` Mode int `yaml:"mode"` Proxies YamlProxies `yaml:"proxies"` Retry int `yaml:"retry"` @@ -110,7 +110,7 @@ type Config struct { // newDefaultConfig create a default config func newDefaultConfig() *Config { conf := &Config{ - Session: 1, + // Session: 1, Mode: 0, Retry: 0, Priority: 10000, diff --git a/person.go b/person.go new file mode 100644 index 0000000..9037e56 --- /dev/null +++ b/person.go @@ -0,0 +1,127 @@ +package imitate + +import ( + "log" + "reflect" + "time" + + "474420502.top/eson/curl2info" +) + +// ITask 继承这个接口的类 +type ITask interface { + Execute() + + SetCrontab(cron string) + + SetLastStatus(status bool) + + SetCurl(Curl *curl2info.CURL) + //GetCurl() *curl2info.CURL + + GetProxies() []string + AppendProxies(proxies ...string) + + TimeUp() bool + NextTime() time.Time +} + +var register = make(map[string]reflect.Type) + +func init() { + +} + +// Register 注册 类型 ITask为样本的类型 +func Register(name string, itype interface{}) { + register[name] = reflect.TypeOf(itype) +} + +func makeRegisterType(name string) interface{} { + result := reflect.New(register[name]).Interface() + return result +} + +// Person 以人为单位 +type Person struct { + Tasks []ITask + Conf *Config +} + +// NewPerson 创建一个人实例 +func NewPerson() *Person { + person := &Person{} + + // person.Conf = NewConfig(conf) + // person.Tasks = SplitTasks(person.Conf) + + return person +} + +// LoadConfig 加载配置 +func (person *Person) LoadConfig(conf string) { + person.Conf = NewConfig(conf) + person.Tasks = splitTasks(person.Conf) +} + +// NewPersonWithConfig 创建一个person +func NewPersonWithConfig(conf string) *Person { + person := NewPerson() + person.LoadConfig(conf) + return person +} + +// SplitTasks 拆开出需求的任务 +func splitTasks(conf *Config) []ITask { + var tasks []ITask + + for _, scurl := range conf.Curls { + curl, err := curl2info.ParseRawCURL(scurl) + if err != nil { + panic(err) + } + + task := makeRegisterType(curl.ITask).(ITask) + + switch conf.Mode { + case 0: + + if curl.Crontab != "" { + task.SetCrontab(curl.Crontab) + } else { + log.Println(conf.Crontab) + task.SetCrontab(conf.Crontab) + } + task.SetCurl(curl) + task.AppendProxies(conf.Proxies...) + tasks = append(tasks, task) + + case 1: + for _, proxy := range conf.Proxies { + ncurl, err := curl2info.ParseRawCURL(scurl) + if err != nil { + panic(err) + } + + ptask := makeRegisterType(ncurl.ITask).(ITask) + if ncurl.Crontab != "" { + ptask.SetCrontab(ncurl.Crontab) + } else { + ptask.SetCrontab(conf.Crontab) + } + InitTask(ptask, ncurl) + + ptask.AppendProxies(proxy) + tasks = append(tasks, task) + } + } + } + return tasks +} + +// Execute 人的执行所有任务 +func (person *Person) Execute() { + for _, task := range person.Tasks { + ExecuteOnPlan(task) + } +} diff --git a/task.go b/task.go index fc177a5..8b4294d 100644 --- a/task.go +++ b/task.go @@ -1,8 +1,6 @@ package imitate import ( - "log" - "reflect" "time" crontab "474420502.top/eson/crontabex" @@ -10,122 +8,6 @@ import ( "474420502.top/eson/requests" ) -var register = make(map[string]reflect.Type) - -func init() { - -} - -// Register 注册 类型 ITask为样本的类型 -func Register(name string, itype interface{}) { - register[name] = reflect.TypeOf(itype) -} - -func makeRegisterType(name string) interface{} { - result := reflect.New(register[name]).Interface() - return result -} - -// Person 以人为单位 -type Person struct { - Tasks []ITask - Conf *Config -} - -// NewPerson 创建一个人实例 -func NewPerson() *Person { - person := &Person{} - - // person.Conf = NewConfig(conf) - // person.Tasks = SplitTasks(person.Conf) - - return person -} - -// LoadConfig 加载配置 -func (person *Person) LoadConfig(conf string) { - person.Conf = NewConfig(conf) - person.Tasks = splitTasks(person.Conf) -} - -// NewPersonWithConfig 创建一个person -func NewPersonWithConfig(conf string) *Person { - person := NewPerson() - person.LoadConfig(conf) - return person -} - -// SplitTasks 拆开出需求的任务 -func splitTasks(conf *Config) []ITask { - var tasks []ITask - - for _, scurl := range conf.Curls { - curl, err := curl2info.ParseRawCURL(scurl) - if err != nil { - panic(err) - } - - task := makeRegisterType(curl.ITask).(ITask) - - switch conf.Mode { - case 0: - - if curl.Crontab != "" { - task.SetCrontab(curl.Crontab) - } else { - log.Println(conf.Crontab) - task.SetCrontab(conf.Crontab) - } - task.SetCurl(curl) - task.AppendProxies(conf.Proxies...) - tasks = append(tasks, task) - - case 1: - for _, proxy := range conf.Proxies { - ncurl, err := curl2info.ParseRawCURL(scurl) - if err != nil { - panic(err) - } - - ptask := makeRegisterType(ncurl.ITask).(ITask) - if ncurl.Crontab != "" { - ptask.SetCrontab(ncurl.Crontab) - } else { - ptask.SetCrontab(conf.Crontab) - } - InitTask(ptask, ncurl) - - ptask.AppendProxies(proxy) - tasks = append(tasks, task) - } - } - } - return tasks -} - -// Execute 人的执行所有任务 -func (person *Person) Execute() { - for _, task := range person.Tasks { - ExecuteOnPlan(task) - } -} - -// ITask 继承这个接口的类 -type ITask interface { - Execute() - - SetCrontab(cron string) - - SetCurl(Curl *curl2info.CURL) - GetCurl() *curl2info.CURL - - GetProxies() []string - AppendProxies(proxies ...string) - - TimeUp() bool - NextTime() time.Time -} - // Task 任务 type Task struct { ITask @@ -133,35 +15,48 @@ type Task struct { crontab *crontab.Crontab curl *curl2info.CURL workflow *requests.Workflow + session *requests.Session proxies YamlProxies } +// SetCurl 设置任务的curl信息类 func (task *Task) SetCurl(curl *curl2info.CURL) { task.curl = curl } -func (task *Task) GetCurl() *curl2info.CURL { - return task.curl -} +// GetCurl 获取任务的curl信息类 +// func (task *Task) GetCurl() *curl2info.CURL { +// return task.curl +// } +// AppendProxies 添加代理集合 func (task *Task) AppendProxies(proxies ...string) { for _, proxy := range proxies { task.proxies = append(task.proxies, proxy) } } +// GetProxies 获取代理的字符串 func (task *Task) GetProxies() []string { return task.proxies } +// SetCrontab 设置crontab的控制规则字符串 func (task *Task) SetCrontab(cron string) { task.crontab = crontab.NewCrontab(cron) } +// SetLastStatus 设置上次执行的状态 成功true 失败false +func (task *Task) SetLastStatus(status bool) { + task.crontab.SetStatus(status) +} + +// TimeUp 判断是否到了下次执行的时间点 func (task *Task) TimeUp() bool { return task.crontab.TimeUp() } +// NextTime 下次执行的时间点 func (task *Task) NextTime() time.Time { return task.crontab.NextTime() } @@ -173,21 +68,38 @@ func InitTask(task ITask, Curl *curl2info.CURL) { // ExecuteOnPlan 按照计划执行任务并返回结果 func ExecuteOnPlan(task ITask) { - log.Println(task) - for i := 0; i < 2; i++ { - + for { if task.TimeUp() { task.Execute() // 事件 在这里变化 } - - nextTime := task.NextTime() - log.Println(nextTime) - time.Sleep(time.Second * 2) + interval := task.NextTime().Sub(time.Now()) + time.Sleep(interval) } } -// Request 根据curl信息执行, TODO: 通用方法设置, 多太实现 +// Workflow 根据现有session获取Workflow的信息, 便于设置或者更改参数, Session持久化 +func (task *Task) Workflow(persistent bool) *requests.Workflow { + if task.workflow == nil { + task.workflow = task.curl.CreateWorkflow(task.Session()) + } + + if persistent { + return task.workflow + } + + return task.curl.CreateWorkflow(nil) +} + +// Request 根据curl信息执行,没持久化 func (task *Task) Request() (*requests.Response, error) { return task.curl.CreateWorkflow(nil).Execute() } + +// Session 获取Session的信息 只保留session的数据和url参数. +func (task *Task) Session() *requests.Session { + if task.session == nil { + task.session = task.curl.CreateSession() + } + return task.session +} diff --git a/task_test.go b/task_test.go index 96000a7..a25b20d 100644 --- a/task_test.go +++ b/task_test.go @@ -2,7 +2,6 @@ package imitate import ( "log" - "reflect" "testing" "474420502.top/eson/curl2info" @@ -36,10 +35,8 @@ func (tt *Toutiao) Execute() { func TestExecutePlan(t *testing.T) { Register("toutiao", Toutiao{}) - person := NewPerson() person.LoadConfig("test.yaml") - log.Println(person.Tasks, reflect.TypeOf(person.Tasks[0]), person.Tasks[0].GetCurl()) person.Execute() t.Error("") diff --git a/test.yaml b/test.yaml index 15b57dc..3f39f36 100644 --- a/test.yaml +++ b/test.yaml @@ -1,4 +1,3 @@ -session : 1 mode : 0 # proxies : "socks5://10.10.10.1:8080" // 支持, 列表 与 单项字符串 proxies : ["socks5://10.10.10.1:8080", "socks5://10.10.10.1:8082", "socks5://10.10.10.1:8083", "socks5://10.10.10.1:8085", "socks5://10.10.10.1:8087", "socks5://10.10.10.1:8088", "socks5://10.10.10.1:8089", "socks5://10.10.10.1:8090"]