eson/imitater
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

test success! and TODO: 添加全局Task指定参数

Browse Source
This commit is contained in:
huangsimin 2018-12-07 12:18:29 +08:00
parent fdd9fb0bf3
commit d08f6c8e68
5 changed files with 170 additions and 135 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
config.go
View File

@ -89,7 +89,7 @@ func (proxies *YamlProxies) MarshalYAML() (interface{}, error) {
// Config 任务加载的默认配置
type Config struct {
Session int `yaml:"session"`
// Session int `yaml:"session"`
Mode int `yaml:"mode"`
Proxies YamlProxies `yaml:"proxies"`
Retry int `yaml:"retry"`
@ -110,7 +110,7 @@ type Config struct {
// newDefaultConfig create a default config
func newDefaultConfig() *Config {
conf := &Config{
Session: 1,
// Session: 1,
Mode: 0,
Retry: 0,
Priority: 10000,

127
person.go Normal file
View File

@ -0,0 +1,127 @@
package imitate
import (
"log"
"reflect"
"time"
"474420502.top/eson/curl2info"
)
// ITask 继承这个接口的类
type ITask interface {
Execute()
SetCrontab(cron string)
SetLastStatus(status bool)
SetCurl(Curl *curl2info.CURL)
//GetCurl() *curl2info.CURL
GetProxies() []string
AppendProxies(proxies ...string)
TimeUp() bool
NextTime() time.Time
}
var register = make(map[string]reflect.Type)
func init() {
}
// Register 注册 类型 ITask为样本的类型
func Register(name string, itype interface{}) {
register[name] = reflect.TypeOf(itype)
}
func makeRegisterType(name string) interface{} {
result := reflect.New(register[name]).Interface()
return result
}
// Person 以人为单位
type Person struct {
Tasks []ITask
Conf *Config
}
// NewPerson 创建一个人实例
func NewPerson() *Person {
person := &Person{}
// person.Conf = NewConfig(conf)
// person.Tasks = SplitTasks(person.Conf)
return person
}
// LoadConfig 加载配置
func (person *Person) LoadConfig(conf string) {
person.Conf = NewConfig(conf)
person.Tasks = splitTasks(person.Conf)
}
// NewPersonWithConfig 创建一个person
func NewPersonWithConfig(conf string) *Person {
person := NewPerson()
person.LoadConfig(conf)
return person
}
// SplitTasks 拆开出需求的任务
func splitTasks(conf *Config) []ITask {
var tasks []ITask
for _, scurl := range conf.Curls {
curl, err := curl2info.ParseRawCURL(scurl)
if err != nil {
panic(err)
}
task := makeRegisterType(curl.ITask).(ITask)
switch conf.Mode {
case 0:
if curl.Crontab != "" {
task.SetCrontab(curl.Crontab)
} else {
log.Println(conf.Crontab)
task.SetCrontab(conf.Crontab)
}
task.SetCurl(curl)
task.AppendProxies(conf.Proxies...)
tasks = append(tasks, task)
case 1:
for _, proxy := range conf.Proxies {
ncurl, err := curl2info.ParseRawCURL(scurl)
if err != nil {
panic(err)
}
ptask := makeRegisterType(ncurl.ITask).(ITask)
if ncurl.Crontab != "" {
ptask.SetCrontab(ncurl.Crontab)
} else {
ptask.SetCrontab(conf.Crontab)
}
InitTask(ptask, ncurl)
ptask.AppendProxies(proxy)
tasks = append(tasks, task)
}
}
}
return tasks
}
// Execute 人的执行所有任务
func (person *Person) Execute() {
for _, task := range person.Tasks {
ExecuteOnPlan(task)
}
}

170
task.go
View File

@ -1,8 +1,6 @@
package imitate
import (
"log"
"reflect"
"time"
crontab "474420502.top/eson/crontabex"
@ -10,122 +8,6 @@ import (
"474420502.top/eson/requests"
)
var register = make(map[string]reflect.Type)
func init() {
}
// Register 注册 类型 ITask为样本的类型
func Register(name string, itype interface{}) {
register[name] = reflect.TypeOf(itype)
}
func makeRegisterType(name string) interface{} {
result := reflect.New(register[name]).Interface()
return result
}
// Person 以人为单位
type Person struct {
Tasks []ITask
Conf *Config
}
// NewPerson 创建一个人实例
func NewPerson() *Person {
person := &Person{}
// person.Conf = NewConfig(conf)
// person.Tasks = SplitTasks(person.Conf)
return person
}
// LoadConfig 加载配置
func (person *Person) LoadConfig(conf string) {
person.Conf = NewConfig(conf)
person.Tasks = splitTasks(person.Conf)
}
// NewPersonWithConfig 创建一个person
func NewPersonWithConfig(conf string) *Person {
person := NewPerson()
person.LoadConfig(conf)
return person
}
// SplitTasks 拆开出需求的任务
func splitTasks(conf *Config) []ITask {
var tasks []ITask
for _, scurl := range conf.Curls {
curl, err := curl2info.ParseRawCURL(scurl)
if err != nil {
panic(err)
}
task := makeRegisterType(curl.ITask).(ITask)
switch conf.Mode {
case 0:
if curl.Crontab != "" {
task.SetCrontab(curl.Crontab)
} else {
log.Println(conf.Crontab)
task.SetCrontab(conf.Crontab)
}
task.SetCurl(curl)
task.AppendProxies(conf.Proxies...)
tasks = append(tasks, task)
case 1:
for _, proxy := range conf.Proxies {
ncurl, err := curl2info.ParseRawCURL(scurl)
if err != nil {
panic(err)
}
ptask := makeRegisterType(ncurl.ITask).(ITask)
if ncurl.Crontab != "" {
ptask.SetCrontab(ncurl.Crontab)
} else {
ptask.SetCrontab(conf.Crontab)
}
InitTask(ptask, ncurl)
ptask.AppendProxies(proxy)
tasks = append(tasks, task)
}
}
}
return tasks
}
// Execute 人的执行所有任务
func (person *Person) Execute() {
for _, task := range person.Tasks {
ExecuteOnPlan(task)
}
}
// ITask 继承这个接口的类
type ITask interface {
Execute()
SetCrontab(cron string)
SetCurl(Curl *curl2info.CURL)
GetCurl() *curl2info.CURL
GetProxies() []string
AppendProxies(proxies ...string)
TimeUp() bool
NextTime() time.Time
}
// Task 任务
type Task struct {
ITask
@ -133,35 +15,48 @@ type Task struct {
crontab *crontab.Crontab
curl *curl2info.CURL
workflow *requests.Workflow
session *requests.Session
proxies YamlProxies
}
// SetCurl 设置任务的curl信息类
func (task *Task) SetCurl(curl *curl2info.CURL) {
task.curl = curl
}
func (task *Task) GetCurl() *curl2info.CURL {
return task.curl
}
// GetCurl 获取任务的curl信息类
// func (task *Task) GetCurl() *curl2info.CURL {
// return task.curl
// }
// AppendProxies 添加代理集合
func (task *Task) AppendProxies(proxies ...string) {
for _, proxy := range proxies {
task.proxies = append(task.proxies, proxy)
}
}
// GetProxies 获取代理的字符串
func (task *Task) GetProxies() []string {
return task.proxies
}
// SetCrontab 设置crontab的控制规则字符串
func (task *Task) SetCrontab(cron string) {
task.crontab = crontab.NewCrontab(cron)
}
// SetLastStatus 设置上次执行的状态 成功true 失败false
func (task *Task) SetLastStatus(status bool) {
task.crontab.SetStatus(status)
}
// TimeUp 判断是否到了下次执行的时间点
func (task *Task) TimeUp() bool {
return task.crontab.TimeUp()
}
// NextTime 下次执行的时间点
func (task *Task) NextTime() time.Time {
return task.crontab.NextTime()
}
@ -173,21 +68,38 @@ func InitTask(task ITask, Curl *curl2info.CURL) {
// ExecuteOnPlan 按照计划执行任务并返回结果
func ExecuteOnPlan(task ITask) {
log.Println(task)
for i := 0; i < 2; i++ {
for {
if task.TimeUp() {
task.Execute() // 事件 在这里变化
}
nextTime := task.NextTime()
log.Println(nextTime)
time.Sleep(time.Second * 2)
interval := task.NextTime().Sub(time.Now())
time.Sleep(interval)
}
}
// Request 根据curl信息执行, TODO: 通用方法设置, 多太实现
// Workflow 根据现有session获取Workflow的信息, 便于设置或者更改参数, Session持久化
func (task *Task) Workflow(persistent bool) *requests.Workflow {
if task.workflow == nil {
task.workflow = task.curl.CreateWorkflow(task.Session())
}
if persistent {
return task.workflow
}
return task.curl.CreateWorkflow(nil)
}
// Request 根据curl信息执行,没持久化
func (task *Task) Request() (*requests.Response, error) {
return task.curl.CreateWorkflow(nil).Execute()
}
// Session 获取Session的信息 只保留session的数据和url参数.
func (task *Task) Session() *requests.Session {
if task.session == nil {
task.session = task.curl.CreateSession()
}
return task.session
}

3
task_test.go
View File

@ -2,7 +2,6 @@ package imitate
import (
"log"
"reflect"
"testing"
"474420502.top/eson/curl2info"
@ -36,10 +35,8 @@ func (tt *Toutiao) Execute() {
func TestExecutePlan(t *testing.T) {
Register("toutiao", Toutiao{})
person := NewPerson()
person.LoadConfig("test.yaml")
log.Println(person.Tasks, reflect.TypeOf(person.Tasks[0]), person.Tasks[0].GetCurl())
person.Execute()
t.Error("")

1
test.yaml
View File

@ -1,4 +1,3 @@
session : 1
mode : 0
# proxies : "socks5://10.10.10.1:8080" // 支持, 列表 与 单项字符串
proxies : ["socks5://10.10.10.1:8080", "socks5://10.10.10.1:8082", "socks5://10.10.10.1:8083", "socks5://10.10.10.1:8085", "socks5://10.10.10.1:8087", "socks5://10.10.10.1:8088", "socks5://10.10.10.1:8089", "socks5://10.10.10.1:8090"]