Compare commits
8 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 478ceec906 | |||
| ab97d012cd | |||
| cff0a27aa0 | |||
|
|
d85d8b3059 | ||
|
|
d6cae41111 | ||
|
|
4fc3a91d35 | ||
|
|
2127db6ce2 | ||
|
|
6f299f87e7 |
26
base_test.go
26
base_test.go
@@ -1,26 +0,0 @@
|
||||
package curl2info
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestPQueue(t *testing.T) {
|
||||
PQExec := newPQueueExecute()
|
||||
PQExec.Push(&parseFunction{Priority: 5})
|
||||
PQExec.Push(&parseFunction{Priority: 10})
|
||||
PQExec.Push(&parseFunction{Priority: 4})
|
||||
PQExec.Push(&parseFunction{Priority: 4})
|
||||
PQExec.Push(&parseFunction{Priority: 20})
|
||||
PQExec.Push(&parseFunction{Priority: 10})
|
||||
PQExec.Push(&parseFunction{Priority: 15})
|
||||
|
||||
content := ""
|
||||
for PQExec.Len() > 0 {
|
||||
content += strconv.Itoa(PQExec.Pop().Priority)
|
||||
content += " "
|
||||
}
|
||||
if content != "4 4 5 10 10 15 20 " {
|
||||
t.Error(content)
|
||||
}
|
||||
}
|
||||
410
crontab.go
410
crontab.go
@@ -1,410 +0,0 @@
|
||||
package curl2info
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Pallinder/go-randomdata"
|
||||
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
)
|
||||
|
||||
type randLR struct {
|
||||
left, right int
|
||||
}
|
||||
|
||||
type hInterval struct {
|
||||
PlanFail []randLR
|
||||
PlanNormal []randLR
|
||||
|
||||
Count int
|
||||
ConstCount int
|
||||
}
|
||||
|
||||
func (interval *hInterval) reset() {
|
||||
interval.Count = interval.ConstCount
|
||||
}
|
||||
|
||||
type timePointer struct {
|
||||
left, right int
|
||||
leftlimit, rightlimit int
|
||||
per int
|
||||
isAll bool
|
||||
}
|
||||
|
||||
func (tp *timePointer) String() string {
|
||||
return fmt.Sprintf("left: %d, right: %d, leftlimit: %d, rightlimit: %d, per: %d", tp.left, tp.right, tp.leftlimit, tp.rightlimit, tp.per)
|
||||
}
|
||||
|
||||
// Crontab 的string解析
|
||||
type Crontab struct {
|
||||
min []timePointer
|
||||
hour []timePointer
|
||||
day []timePointer
|
||||
month []timePointer
|
||||
week []timePointer
|
||||
|
||||
WillPlans []time.Time
|
||||
SkipPlans []time.Time
|
||||
|
||||
YearPlan *trieYear
|
||||
|
||||
interval *CircularLinked
|
||||
lastStatus bool
|
||||
nextTime time.Time
|
||||
}
|
||||
|
||||
// NewCrontab create 一个crontab
|
||||
func NewCrontab(crontab string) *Crontab {
|
||||
cron := &Crontab{}
|
||||
cron.FromString(crontab)
|
||||
return cron
|
||||
}
|
||||
|
||||
// SetStatus 设置状态 接口定义
|
||||
func (cron *Crontab) SetStatus(status interface{}) {
|
||||
if cron.interval != nil {
|
||||
cron.lastStatus = status.(bool)
|
||||
}
|
||||
}
|
||||
|
||||
// GetStatus 设置状态 接口定义
|
||||
func (cron *Crontab) GetStatus() (status interface{}) {
|
||||
if cron.interval != nil {
|
||||
return cron.lastStatus
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// TimeUp 是否时间快到
|
||||
func (cron *Crontab) TimeUp() bool {
|
||||
|
||||
if cron.interval != nil {
|
||||
return cron.intervalTimeUp()
|
||||
}
|
||||
|
||||
return cron.linuxTimeUp()
|
||||
}
|
||||
|
||||
// NextTime 返回下次任务的时间
|
||||
func (cron *Crontab) NextTime() *time.Time {
|
||||
if cron.interval != nil {
|
||||
return &cron.nextTime
|
||||
}
|
||||
|
||||
if len(cron.WillPlans) > 0 {
|
||||
return &cron.WillPlans[0]
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (cron *Crontab) String() string {
|
||||
return fmt.Sprintf("min:%s\nhour:%s\nday:%s\nmonth:%s\nweek:%s\n", spew.Sdump(cron.min), spew.Sdump(cron.hour), spew.Sdump(cron.day), spew.Sdump(cron.month), spew.Sdump(cron.week))
|
||||
}
|
||||
|
||||
// FromString 解析crontab 的 表达式
|
||||
func (cron *Crontab) FromString(crontab string) error {
|
||||
crontab = strings.TrimSpace(crontab)
|
||||
|
||||
matches := regexp.MustCompile("[^ ]+").FindAllString(crontab, -1)
|
||||
mlen := len(matches)
|
||||
switch mlen {
|
||||
case 1:
|
||||
// "f1-2|5-10x5,f1|10m,10-15,f1"
|
||||
cron.nextTime = time.Now()
|
||||
cron.lastStatus = true
|
||||
cron.interval = NewCircularLinked()
|
||||
var intervalList []interface{}
|
||||
intervalList = parseIntervalString(matches[0])
|
||||
cron.interval.Append(intervalList...)
|
||||
|
||||
case 5:
|
||||
cron.min = createTimePointer(matches[0], 0, 59, true)
|
||||
cron.hour = createTimePointer(matches[1], 0, 23, true)
|
||||
cron.day = createTimePointer(matches[2], 1, 31, false)
|
||||
cron.month = createTimePointer(matches[3], 1, 12, true)
|
||||
cron.week = createTimePointer(matches[4], 0, 6, true)
|
||||
|
||||
cron.createYearPlan()
|
||||
default:
|
||||
return errors.New("mathches len != want, check crontab string")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// createYearPlan 创建年度计划
|
||||
func (cron *Crontab) createYearPlan() {
|
||||
cron.YearPlan = newTrieYear()
|
||||
cron.YearPlan.FromCrontab(cron)
|
||||
}
|
||||
|
||||
func (cron *Crontab) linuxTimeUp() bool {
|
||||
now := time.Now()
|
||||
maxlen := 1000
|
||||
createlen := 500
|
||||
|
||||
plen := len(cron.WillPlans)
|
||||
if plen <= createlen {
|
||||
var lastplan time.Time
|
||||
if plen == 0 {
|
||||
lastplan = now
|
||||
} else {
|
||||
lastplan = cron.WillPlans[plen-1].Add(time.Minute)
|
||||
}
|
||||
if !cron.YearPlan.CheckYear() {
|
||||
cron.createYearPlan()
|
||||
}
|
||||
|
||||
timeplans := cron.YearPlan.GetPlanTime(cron, lastplan, uint(maxlen-plen))
|
||||
cron.WillPlans = append(cron.WillPlans, timeplans...)
|
||||
}
|
||||
|
||||
if len(cron.WillPlans) > 0 {
|
||||
istimeup := false
|
||||
for i := 0; i < maxlen; i++ {
|
||||
if now.Unix() >= cron.WillPlans[i].Unix() {
|
||||
istimeup = true
|
||||
} else {
|
||||
if istimeup {
|
||||
if i-1 > 0 {
|
||||
cron.SkipPlans = append(cron.SkipPlans, cron.WillPlans[0:i-1]...)
|
||||
if len(cron.SkipPlans) >= maxlen+200 {
|
||||
cron.SkipPlans = cron.SkipPlans[200:]
|
||||
}
|
||||
}
|
||||
cron.WillPlans = cron.WillPlans[i:]
|
||||
return istimeup
|
||||
}
|
||||
|
||||
return istimeup
|
||||
}
|
||||
}
|
||||
|
||||
cron.SkipPlans = append(cron.SkipPlans, cron.WillPlans...)
|
||||
cron.WillPlans = nil
|
||||
return istimeup
|
||||
}
|
||||
|
||||
log.Panicln("error willplans range")
|
||||
return false
|
||||
}
|
||||
|
||||
func (cron *Crontab) intervalTimeUp() bool {
|
||||
|
||||
now := time.Now()
|
||||
if now.Unix() >= cron.nextTime.Unix() {
|
||||
|
||||
iv := cron.interval.Cursor().GetValue().(hInterval)
|
||||
isecond := 0
|
||||
if cron.lastStatus == false && len(iv.PlanFail) > 0 {
|
||||
idx := randomdata.Number(len(iv.PlanFail))
|
||||
lr := iv.PlanFail[idx]
|
||||
isecond = randomdata.Number(lr.left, lr.right+1)
|
||||
} else {
|
||||
idx := randomdata.Number(len(iv.PlanNormal))
|
||||
lr := iv.PlanNormal[idx]
|
||||
isecond = randomdata.Number(lr.left, lr.right+1)
|
||||
}
|
||||
|
||||
iv.Count--
|
||||
if iv.Count <= 0 {
|
||||
iv.reset()
|
||||
cron.interval.MoveNext()
|
||||
}
|
||||
|
||||
cron.nextTime = now.Add(time.Duration(isecond) * time.Second)
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func createTimePointer(min string, llimit, rlimit int, fixedLeftRight bool) []timePointer {
|
||||
|
||||
var result []timePointer
|
||||
|
||||
exelist := strings.Split(min, ",")
|
||||
for _, exe := range exelist {
|
||||
tp := timePointer{}
|
||||
|
||||
takeper := strings.Split(exe, "/") // per
|
||||
var rangevalue, per string
|
||||
if len(takeper) == 1 {
|
||||
rangevalue = exe
|
||||
per = "1"
|
||||
} else {
|
||||
rangevalue = takeper[0]
|
||||
per = takeper[1]
|
||||
}
|
||||
// takeRange
|
||||
be := strings.Split(rangevalue, "-")
|
||||
var left, rigth string
|
||||
switch len(be) {
|
||||
case 1:
|
||||
left = be[0]
|
||||
rigth = be[0]
|
||||
case 2:
|
||||
left = be[0]
|
||||
rigth = be[1]
|
||||
default:
|
||||
panic(errors.New("range value is > 2"))
|
||||
}
|
||||
|
||||
if left == "*" {
|
||||
tp.left = llimit
|
||||
} else {
|
||||
ileft, err := strconv.Atoi(strings.Replace(left, "^", "-", -1))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
tp.left = ileft
|
||||
}
|
||||
|
||||
if rigth == "*" {
|
||||
tp.right = rlimit
|
||||
} else {
|
||||
iright, err := strconv.Atoi(strings.Replace(rigth, "^", "-", -1))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
tp.right = iright
|
||||
}
|
||||
|
||||
iper, err := strconv.Atoi(per)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
tp.per = iper
|
||||
tp.leftlimit = llimit
|
||||
tp.rightlimit = rlimit
|
||||
|
||||
// 修正左值
|
||||
leftfixed := tp.left
|
||||
if leftfixed < 0 {
|
||||
leftfixed += tp.rightlimit + 1
|
||||
if fixedLeftRight {
|
||||
tp.left = leftfixed
|
||||
}
|
||||
}
|
||||
|
||||
rightfixed := tp.right
|
||||
if rightfixed < 0 {
|
||||
rightfixed += tp.rightlimit + 1
|
||||
if fixedLeftRight {
|
||||
tp.right = rightfixed
|
||||
}
|
||||
}
|
||||
|
||||
// 全部符合 当左等于左 且 右等于右最大 并且 per == 1 TODO: 如果加入时间间隔 就需要 加多一个 附加值
|
||||
if leftfixed == tp.leftlimit && rightfixed == tp.rightlimit && tp.per == 1 {
|
||||
tp.isAll = true
|
||||
}
|
||||
|
||||
result = append(result, tp)
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
func parseIntervalString(crontab string) []interface{} {
|
||||
var result []interface{}
|
||||
|
||||
values := strings.Split(crontab, ",")
|
||||
for _, value := range values {
|
||||
interval := hInterval{}
|
||||
// 次数
|
||||
valuesCounts := strings.Split(value, "x")
|
||||
switch len(valuesCounts) {
|
||||
case 1:
|
||||
interval.ConstCount = 1
|
||||
|
||||
case 2:
|
||||
count, err := strconv.Atoi(valuesCounts[1])
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
interval.ConstCount = count
|
||||
default:
|
||||
panic("valuesCounts error, the len is not in range")
|
||||
}
|
||||
|
||||
// 统计失败与普通间隔值的数组
|
||||
failAndNormal := valuesCounts[0]
|
||||
|
||||
valuesFN := strings.Split(failAndNormal, "|")
|
||||
|
||||
for _, FN := range valuesFN {
|
||||
if FN == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
switch FN[0] {
|
||||
case 'f', 'F':
|
||||
fvalue := FN[1:]
|
||||
interval.PlanFail = append(interval.PlanFail, parseRandLR(fvalue))
|
||||
case 'n':
|
||||
value := FN[1:]
|
||||
interval.PlanNormal = append(interval.PlanNormal, parseRandLR(value))
|
||||
default:
|
||||
interval.PlanNormal = append(interval.PlanNormal, parseRandLR(FN))
|
||||
}
|
||||
}
|
||||
|
||||
interval.reset()
|
||||
result = append(result, interval)
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
func parseRandLR(value string) randLR {
|
||||
vlen := len(value)
|
||||
lastchar := value[vlen-1]
|
||||
|
||||
lr := strings.Split(value, "-")
|
||||
switch len(lr) {
|
||||
case 1:
|
||||
lr := randLR{parseTimeValue(lr[0], lastchar), parseTimeValue(lr[0], lastchar)}
|
||||
return lr
|
||||
case 2:
|
||||
|
||||
lr := randLR{parseTimeValue(lr[0], lastchar), parseTimeValue(lr[1], lastchar)}
|
||||
return lr
|
||||
default:
|
||||
panic("lr is error")
|
||||
}
|
||||
}
|
||||
|
||||
func getInt(v string) int {
|
||||
vint, err := strconv.Atoi(v)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return vint
|
||||
}
|
||||
|
||||
func parseTimeValue(v string, lastchar byte) int {
|
||||
|
||||
vlen := len(v)
|
||||
|
||||
switch lastchar {
|
||||
case 's':
|
||||
return getInt(v[:vlen-1])
|
||||
case 'm':
|
||||
return getInt(v[:vlen-1]) * 60
|
||||
case 'h':
|
||||
return getInt(v[:vlen-1]) * 3600
|
||||
case 'd':
|
||||
return getInt(v[:vlen-1]) * 3600 * 24
|
||||
default:
|
||||
return getInt(v)
|
||||
}
|
||||
}
|
||||
@@ -1,75 +0,0 @@
|
||||
package curl2info
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"runtime"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// type LRValue struct {
|
||||
// left, right int
|
||||
// }
|
||||
|
||||
func TestParseCrontab(t *testing.T) {
|
||||
// crontab := "0-5/2,7-30/3,30,35,40-^1 * * * *" //(秒) 分 时 号(每月的多少号, 要注意月可可能性) 星期几(每个星期的) /每 ,列表 -范围
|
||||
crontab := "* * * * *"
|
||||
|
||||
PrintMemUsage()
|
||||
|
||||
ty := newTrieYear()
|
||||
cron := NewCrontab(crontab)
|
||||
ty.FromCrontab(cron)
|
||||
|
||||
if len(ty.GetPlanTime(cron, time.Now(), 10)) != 10 {
|
||||
t.Error("GetPlanTime error len != 10")
|
||||
}
|
||||
|
||||
cron.createYearPlan()
|
||||
if !cron.TimeUp() {
|
||||
t.Error("timeup error")
|
||||
}
|
||||
|
||||
PrintMemUsage()
|
||||
|
||||
}
|
||||
|
||||
func TestParseInterval(t *testing.T) {
|
||||
// crontab := "0-5/2,7-30/3,30,35,40-^1 * * * *" //(秒) 分 时 号(每月的多少号, 要注意月可可能性) 星期几(每个星期的) /每 ,列表 -范围
|
||||
// t.Error("")
|
||||
// crontab := "f1-2|1-3|5-8x5,f1|10m,10-15,f1"
|
||||
|
||||
// PrintMemUsage()
|
||||
// cron := NewCrontab(crontab)
|
||||
|
||||
// now := time.Now()
|
||||
// for i := 0; i <= 15; i++ {
|
||||
// if cron.TimeUp() {
|
||||
// log.Println(time.Since(now))
|
||||
// now = time.Now()
|
||||
// }
|
||||
// time.Sleep(time.Second)
|
||||
// }
|
||||
|
||||
// PrintMemUsage()
|
||||
}
|
||||
|
||||
func TestCrontabPlus(t *testing.T) {
|
||||
// crontab := "0-5/2,7-30/3,30,35,40-^1 * * * *" //(秒) 分 时 号(每月的多少号, 要注意月可可能性) 星期几(每个星期的) /每 ,列表 -范围
|
||||
// crondata := NewCrontab("*22 * * * *")
|
||||
|
||||
}
|
||||
|
||||
func PrintMemUsage() {
|
||||
var m runtime.MemStats
|
||||
runtime.ReadMemStats(&m)
|
||||
// For info on each, see: https://golang.org/pkg/runtime/#MemStats
|
||||
fmt.Printf("Alloc = %v MiB", bToMb(m.Alloc))
|
||||
fmt.Printf("\tTotalAlloc = %v MiB", bToMb(m.TotalAlloc))
|
||||
fmt.Printf("\tSys = %v MiB", bToMb(m.Sys))
|
||||
fmt.Printf("\tNumGC = %v\n", m.NumGC)
|
||||
}
|
||||
|
||||
func bToMb(b uint64) uint64 {
|
||||
return b / 1024 / 1024
|
||||
}
|
||||
17
option.go
17
option.go
@@ -36,6 +36,7 @@ func init() {
|
||||
// 自定义
|
||||
{"--task", 10, parseITask, &extract{re: "--task +(.+)", execute: extractData}},
|
||||
{"--crontab", 10, parseCrontab, &extract{re: "--crontab +(.+)", execute: extractData}},
|
||||
{"--name", 10, parseName, &extract{re: "--name +(.+)", execute: extractData}},
|
||||
}
|
||||
|
||||
for _, oe := range oelist {
|
||||
@@ -63,7 +64,7 @@ type optionExecute struct {
|
||||
|
||||
Priority int
|
||||
|
||||
Execute func(*CURL, string) // 执行函数
|
||||
Parse func(*CURL, string) // 执行函数
|
||||
Extract *extract // 提取的方法结构与参数
|
||||
}
|
||||
|
||||
@@ -76,27 +77,31 @@ func (oe *optionExecute) BuildFunction(curl *CURL, soption string) *parseFunctio
|
||||
if oe.Extract != nil {
|
||||
data = oe.Extract.Execute(data)
|
||||
}
|
||||
return &parseFunction{ParamCURL: curl, ParamData: data, ExecuteFunction: oe.Execute, Priority: oe.Priority}
|
||||
return &parseFunction{ParamCURL: curl, ParamData: data, ExecuteFunction: oe.Parse, Priority: oe.Priority}
|
||||
}
|
||||
|
||||
func judgeOptions(u *CURL, soption string) *parseFunction {
|
||||
word := trieStrWord(soption)
|
||||
if ioe := optionTrie.SearchMostPrefix(&word); ioe != nil {
|
||||
if ioe := optionTrie.SearchDepth(&word); ioe != nil {
|
||||
oe := ioe.(*optionExecute)
|
||||
return oe.BuildFunction(u, soption)
|
||||
}
|
||||
|
||||
log.Println(soption, " no haved this option")
|
||||
log.Println(soption, " not this option")
|
||||
return nil
|
||||
}
|
||||
|
||||
func extractData(re, soption string) string {
|
||||
datas := regexp.MustCompile(re).FindStringSubmatch(soption)
|
||||
return strings.Trim(datas[1], "'")
|
||||
return strings.Trim(datas[1], "'\"")
|
||||
}
|
||||
|
||||
func parseName(u *CURL, value string) {
|
||||
u.Name = value
|
||||
}
|
||||
|
||||
func parseCrontab(u *CURL, value string) {
|
||||
|
||||
u.Crontab = value
|
||||
}
|
||||
|
||||
func parseITask(u *CURL, value string) {
|
||||
|
||||
@@ -24,6 +24,8 @@ type CURL struct {
|
||||
Insecure bool
|
||||
|
||||
ITask string
|
||||
Crontab string
|
||||
Name string
|
||||
}
|
||||
|
||||
// New new 一个 curl 出来
|
||||
@@ -96,18 +98,14 @@ func (curl *CURL) CreateWorkflow(ses *requests.Session) *requests.Workflow {
|
||||
}
|
||||
|
||||
// ParseRawCURL curl_bash 可以用trie改进 没空改
|
||||
func ParseRawCURL(scurl string) (cURL *CURL, err error) {
|
||||
|
||||
defer func() {
|
||||
if _err := recover(); _err != nil {
|
||||
cURL = nil
|
||||
err = _err.(error)
|
||||
}
|
||||
}()
|
||||
|
||||
func ParseRawCURL(scurl string) (cURL *CURL) {
|
||||
executor := newPQueueExecute()
|
||||
curl := New()
|
||||
|
||||
if len(scurl) <= 4 {
|
||||
panic("scurl error:" + scurl)
|
||||
}
|
||||
|
||||
if scurl[0] == '"' && scurl[len(scurl)-1] == '"' {
|
||||
scurl = strings.Trim(scurl, `"`)
|
||||
} else if scurl[0] == '\'' && scurl[len(scurl)-1] == '\'' {
|
||||
@@ -151,5 +149,5 @@ func ParseRawCURL(scurl string) (cURL *CURL, err error) {
|
||||
curl.Method = "GET"
|
||||
}
|
||||
|
||||
return curl, nil
|
||||
return curl
|
||||
}
|
||||
|
||||
@@ -24,23 +24,18 @@ func TestParseCURL(t *testing.T) {
|
||||
// Access-Control-Request-Method 方法告诉 --data-binary 默认是POST
|
||||
|
||||
for _, scurl := range scurls {
|
||||
curl, err := ParseRawCURL(scurl)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
} else {
|
||||
curl := ParseRawCURL(scurl)
|
||||
|
||||
if curl.Method == "" {
|
||||
t.Error("curl.Method is nil")
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
func TestTouTiaoCURL(t *testing.T) {
|
||||
scurl := "curl 'http://is.snssdk.com/2/article/information/v24/?latitude=22.831367&longitude=113.511515&group_id=6565653745026204168&item_id=6565653745026204168&aggr_type=1&context=1&from_category=news_game&article_page=0&iid=34903754482&device_id=41148471494&ac=wifi&channel=oppo-cpa&aid=13&app_name=news_article&version_code=676&version_name=6.7.6&device_platform=android&ab_version=304489%2C261579%2C373245%2C360501%2C374617%2C366851%2C356335%2C345191%2C271178%2C357704%2C326524%2C326532%2C292723%2C366036%2C323233%2C371779%2C346557%2C351090%2C319958%2C372620%2C362184%2C214069%2C31643%2C333971%2C366873%2C374962%2C372618%2C280449%2C281298%2C366489%2C325619%2C373770%2C357402%2C361073%2C362402%2C290191%2C370014%2C353484%2C375739%2C373725%2C295827%2C353305%2C375426%2C374426%2C239095%2C360541%2C344347%2C170988%2C371590%2C368831%2C368827%2C368775%2C374117%2C365053%2C374232%2C368303%2C375692%2C330632%2C297059%2C374250%2C276206%2C286212%2C350193%2C365036%2C373741%2C374405%2C373368%2C370846%2C364453%2C375713%2C369501%2C369165%2C368839%2C375433%2C373123%2C371555%2C371963%2C374142%2C372907&ab_client=a1%2Cc4%2Ce1%2Cf1%2Cg2%2Cf7&ab_group=94567%2C102754%2C181430&ab_feature=94567%2C102754&abflag=3&ssmix=a&device_type=ONEPLUS+A3010&device_brand=OnePlus&language=zh&os_api=26&os_version=8.0.0&uuid=864854034514328&openudid=9b35a4035eecee2c&manifest_version_code=676&resolution=1080*1920&dpi=420&update_version_code=67610&_rticket=1528706910264&plugin=10603&pos=5r_-9Onkv6e_eCQieCoDeCUfv7G_8fLz-vTp6Pn4v6esrK6zqKysqKyosb_x_On06ej5-L-nr6-zpa6srquqsb_88Pzt3vTp5L-nv3gkIngqA3glH7-xv_zw_O3R8vP69Ono-fi_p6ysrrOupauqqaSxv_zw_O3R_On06ej5-L-nr66zrairpKqv4A%3D%3D&fp=HrT_FlD_PMcIFlD5FSU1FYmeFrxO&rom_version=26&ts=1528706911&as=a265e371dff53b57de5999&mas=0073e8ef3f9a8b842da0ead7d35c0597ea2ee0ccce5e5d5db5' -H 'Accept-Encoding: gzip' -H 'X-SS-REQ-TICKET: 1528706910267' -H 'User-Agent: Dalvik/2.1.0 (Linux; U; Android 8.0.0; ONEPLUS A3010 Build/OPR1.170623.032) NewsArticle/6.7.6 okhttp/3.10.0.1' -H 'Cookie: odin_tt=210899a257b5fe787a3465e2220fb94d91d5ad34c77dee3560f93fccc82dd738cccb301770f633530fdd6ceea955983d; UM_distinctid=163ace3b0050-08fccf530af621-f1c0e26-49a10-163ace3b0093e8; CNZZDATA1271720685=1435124261-1527612007-%7C1527612007; CNZZDATA1264530760=119491224-1527609979-%7C1527612115; JSESSIONID=67814B7DDE08D5A9F3B3D684220CF3FB; alert_coverage=6; qh[360]=1; install_id=34903754482; ttreq=1$b7221ef01bd5ed7c030f5db45e959686c9ddd0d2' -H 'Host: is.snssdk.com' -H 'Connection: Keep-Alive'"
|
||||
curl, err := ParseRawCURL(scurl)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
curl := ParseRawCURL(scurl)
|
||||
|
||||
ses := curl.CreateSession()
|
||||
wf := curl.CreateWorkflow(ses)
|
||||
@@ -56,10 +51,7 @@ func TestTouTiaoCURL(t *testing.T) {
|
||||
|
||||
func TestErrorCurl(t *testing.T) {
|
||||
scurl := `curl 'https://appgrowing.cn/'-H 'authority: appgrowing.cn' -H 'cache-control: max-age=0' -H 'upgrade-insecure-requests: 1' -H 'user-agent: Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1' -H 'accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8' -H 'accept-encoding: gzip, deflate, br' -H 'accept-language: zh' -H 'cookie: _ga=GA1.2.1371058419.1533104518; _gid=GA1.2.896241740.1543307916; _gat_gtag_UA_4002880_19=1' -H 'if-none-match: W/"5bf7a0a9-ca6"' -H 'if-modified-since: Fri, 23 Nov 2018 06:39:37 GMT'`
|
||||
curl, err := ParseRawCURL(scurl)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
curl := ParseRawCURL(scurl)
|
||||
|
||||
ses := curl.CreateSession()
|
||||
wf := curl.CreateWorkflow(ses)
|
||||
@@ -75,14 +67,11 @@ func TestErrorCurl(t *testing.T) {
|
||||
|
||||
func TestCurlTimeout(t *testing.T) {
|
||||
scurl := `curl 'https://javtc.com/' --connect-timeout 1 -H 'authority: appgrowing.cn' -H 'cache-control: max-age=0' -H 'upgrade-insecure-requests: 1' -H 'user-agent: Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1' -H 'accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8' -H 'accept-encoding: gzip, deflate, br' -H 'accept-language: zh' -H 'cookie: _ga=GA1.2.1371058419.1533104518; _gid=GA1.2.896241740.1543307916; _gat_gtag_UA_4002880_19=1' -H 'if-none-match: W/"5bf7a0a9-ca6"' -H 'if-modified-since: Fri, 23 Nov 2018 06:39:37 GMT'`
|
||||
curl, err := ParseRawCURL(scurl)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
curl := ParseRawCURL(scurl)
|
||||
|
||||
ses := curl.CreateSession()
|
||||
wf := curl.CreateWorkflow(ses)
|
||||
_, err = wf.Execute()
|
||||
_, err := wf.Execute()
|
||||
if err == nil {
|
||||
t.Error("not timeout")
|
||||
}
|
||||
|
||||
@@ -110,7 +110,7 @@ func (trie *hTrie) Remove(word string) {
|
||||
}
|
||||
|
||||
// SearchMostPrefix Returns if the word is in the trie.
|
||||
func (trie *hTrie) SearchMostPrefix(iword trieWord) interface{} {
|
||||
func (trie *hTrie) SearchDepth(iword trieWord) interface{} {
|
||||
cur := trie
|
||||
word := iword.GetWord()
|
||||
|
||||
@@ -123,6 +123,8 @@ func (trie *hTrie) SearchMostPrefix(iword trieWord) interface{} {
|
||||
cur = next
|
||||
if cur.isWord {
|
||||
result = cur.value
|
||||
} else {
|
||||
result = nil
|
||||
}
|
||||
} else {
|
||||
return result
|
||||
|
||||
@@ -1 +1,26 @@
|
||||
package curl2info
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestPQueue(t *testing.T) {
|
||||
PQExec := newPQueueExecute()
|
||||
PQExec.Push(&parseFunction{Priority: 5})
|
||||
PQExec.Push(&parseFunction{Priority: 10})
|
||||
PQExec.Push(&parseFunction{Priority: 4})
|
||||
PQExec.Push(&parseFunction{Priority: 4})
|
||||
PQExec.Push(&parseFunction{Priority: 20})
|
||||
PQExec.Push(&parseFunction{Priority: 10})
|
||||
PQExec.Push(&parseFunction{Priority: 15})
|
||||
|
||||
content := ""
|
||||
for PQExec.Len() > 0 {
|
||||
content += strconv.Itoa(PQExec.Pop().Priority)
|
||||
content += " "
|
||||
}
|
||||
if content != "4 4 5 10 10 15 20 " {
|
||||
t.Error(content)
|
||||
}
|
||||
}
|
||||
|
||||
269
trie_year.go
269
trie_year.go
@@ -1,269 +0,0 @@
|
||||
package curl2info
|
||||
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
|
||||
type minuteNode struct {
|
||||
}
|
||||
|
||||
func newMinuteNode() *minuteNode {
|
||||
return &minuteNode{}
|
||||
}
|
||||
|
||||
type hourNode struct {
|
||||
Minute [60]*minuteNode
|
||||
}
|
||||
|
||||
func (hour *hourNode) CreateMinute(nminute int) {
|
||||
min := &minuteNode{}
|
||||
hour.Minute[nminute] = min
|
||||
}
|
||||
|
||||
func newHourNode() *hourNode {
|
||||
return &hourNode{}
|
||||
}
|
||||
|
||||
type dayNode struct {
|
||||
IsClear bool
|
||||
Week time.Weekday
|
||||
Hour [24]*hourNode
|
||||
}
|
||||
|
||||
func (day *dayNode) CreateHour(nhour int) {
|
||||
hour := &hourNode{}
|
||||
day.Hour[nhour] = hour
|
||||
}
|
||||
|
||||
func newDayNode(curday *time.Time) *dayNode {
|
||||
day := &dayNode{}
|
||||
|
||||
week := curday.Weekday()
|
||||
day.Week = week
|
||||
day.IsClear = true
|
||||
|
||||
return day
|
||||
}
|
||||
|
||||
type monthNode struct {
|
||||
MaxDay int
|
||||
First *time.Time
|
||||
Day [32]*dayNode
|
||||
}
|
||||
|
||||
func (month *monthNode) CreateDay(nday int) {
|
||||
day := month.First.AddDate(0, 0, nday-1)
|
||||
month.Day[nday] = newDayNode(&day)
|
||||
}
|
||||
|
||||
func newMonthNode(year, month int) *monthNode {
|
||||
Month := &monthNode{}
|
||||
First := time.Date(year, time.Month(month), 1, 0, 0, 0, 0, time.Local)
|
||||
Month.First = &First
|
||||
Month.MaxDay = Month.First.AddDate(0, 1, -1).Day()
|
||||
return Month
|
||||
}
|
||||
|
||||
type trieYear struct {
|
||||
Year int
|
||||
Month [13]*monthNode
|
||||
}
|
||||
|
||||
// CheckYear 跨年判断
|
||||
func (ty *trieYear) CheckYear() bool {
|
||||
year := time.Now().Year()
|
||||
return ty.Year == year
|
||||
}
|
||||
|
||||
func (ty *trieYear) clearHour() {
|
||||
|
||||
for _, month := range ty.Month {
|
||||
if month != nil {
|
||||
for _, day := range month.Day {
|
||||
if day != nil {
|
||||
for i := 0; i <= 23; i++ {
|
||||
day.Hour[i] = nil
|
||||
day.IsClear = true
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func newTrieYear() *trieYear {
|
||||
ty := trieYear{}
|
||||
ty.Year = time.Now().Year()
|
||||
return &ty
|
||||
}
|
||||
|
||||
// GetPlanTime 获取计划表
|
||||
func (ty *trieYear) GetPlanTime(cron *Crontab, aftertime time.Time, count uint) []time.Time {
|
||||
|
||||
now := aftertime
|
||||
nmonth := int(now.Month())
|
||||
nday := now.Day()
|
||||
nhour := now.Hour()
|
||||
nminute := now.Minute()
|
||||
|
||||
var result []time.Time
|
||||
|
||||
for i := 1; i <= 12; i++ {
|
||||
|
||||
if i < nmonth {
|
||||
continue
|
||||
}
|
||||
|
||||
month := ty.Month[i]
|
||||
if month != nil {
|
||||
|
||||
for j := 1; j <= 31; j++ {
|
||||
|
||||
if nmonth == i {
|
||||
if j < nday {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
day := month.Day[j]
|
||||
if day != nil {
|
||||
|
||||
if day.IsClear {
|
||||
insertHour(cron, day)
|
||||
}
|
||||
|
||||
for k := 0; k <= 23; k++ {
|
||||
|
||||
if nmonth == i && nday == j {
|
||||
if k < nhour {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
hour := day.Hour[k]
|
||||
|
||||
if hour != nil {
|
||||
|
||||
for n := 0; n <= 59; n++ {
|
||||
if nmonth == i && nday == j && nhour == k {
|
||||
if n < nminute {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
min := hour.Minute[n]
|
||||
if min != nil {
|
||||
|
||||
curTime := time.Date(ty.Year, time.Month(i), j, k, n, 0, 0, time.Local)
|
||||
result = append(result, curTime)
|
||||
count--
|
||||
if count <= 0 {
|
||||
ty.clearHour()
|
||||
return result
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ty.clearHour()
|
||||
return result
|
||||
}
|
||||
|
||||
// FromCrontab 从Crontab生成树
|
||||
func (ty *trieYear) FromCrontab(cron *Crontab) {
|
||||
// 月的填充
|
||||
for _, month := range cron.month {
|
||||
|
||||
left := month.left
|
||||
right := month.right
|
||||
|
||||
for i := left; i <= right; i += month.per {
|
||||
curMonth := newMonthNode(ty.Year, i)
|
||||
ty.Month[i] = curMonth
|
||||
// 天的填充
|
||||
insertDay(cron, curMonth)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func filterDay(cron *Crontab, curday *dayNode) bool {
|
||||
|
||||
for _, w := range cron.week {
|
||||
if w.isAll {
|
||||
return false
|
||||
}
|
||||
|
||||
for n := w.left; n <= w.right; n += w.per {
|
||||
if n == int(curday.Week) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func insertDay(cron *Crontab, curMonth *monthNode) {
|
||||
for _, day := range cron.day {
|
||||
|
||||
left := day.left
|
||||
if left < 0 {
|
||||
left += curMonth.MaxDay + 1
|
||||
}
|
||||
|
||||
right := day.right
|
||||
if right < 0 {
|
||||
right += curMonth.MaxDay + 1
|
||||
}
|
||||
|
||||
for j := left; j <= right; j += day.per {
|
||||
curMonth.CreateDay(j)
|
||||
curDay := curMonth.Day[j]
|
||||
|
||||
if filterDay(cron, curDay) {
|
||||
curMonth.Day[j] = nil
|
||||
} else {
|
||||
// insertHour(cron, curDay)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
func insertHour(cron *Crontab, curDay *dayNode) {
|
||||
curDay.IsClear = false
|
||||
// 时的填充
|
||||
for _, hour := range cron.hour {
|
||||
|
||||
left := hour.left
|
||||
right := hour.right
|
||||
|
||||
for k := left; k <= right; k += hour.per {
|
||||
curDay.CreateHour(k)
|
||||
curHour := curDay.Hour[k]
|
||||
insertMinute(cron, curHour)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func insertMinute(cron *Crontab, curHour *hourNode) {
|
||||
for _, min := range cron.min {
|
||||
|
||||
left := min.left
|
||||
right := min.right
|
||||
|
||||
for l := left; l <= right; l += min.per {
|
||||
curHour.CreateMinute(l)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user