diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a6c57f5 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*.json diff --git a/main.go b/main.go index a1df3ec..f2e92c7 100644 --- a/main.go +++ b/main.go @@ -4,13 +4,17 @@ import ( "bytes" "context" "encoding/csv" + "encoding/gob" "io/ioutil" "log" + "os" "regexp" "strconv" + "strings" "time" "github.com/474420502/gcurl" + "github.com/tidwall/gjson" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" "golang.org/x/text/encoding/simplifiedchinese" @@ -37,46 +41,94 @@ type Stock struct { Code int `json:"股票数字代码" bson:"股票数字代码"` } +type StockCode struct { + CodeStr string // 代地区码 + Code string // 不带地区码 +} + func main() { + + log.SetFlags(log.Llongfile | log.LstdFlags) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() - client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://localhost:27017")) + // client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://localhost:27017")) + client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://root:6601502@localhost:27017")) + if err != nil { + panic(err) + } + murl := `curl 'http://quotes.money.163.com/hs/service/diyrank.php?host=http%3A%2F%2Fquotes.money.163.com%2Fhs%2Fservice%2Fdiyrank.php&page=1&query=STYPE%3AEQA&fields=NO%2CSYMBOL%2CNAME%2CPRICE%2CPERCENT%2CUPDOWN%2CFIVE_MINUTE%2COPEN%2CYESTCLOSE%2CHIGH%2CLOW%2CVOLUME%2CTURNOVER%2CHS%2CLB%2CWB%2CZF%2CPE%2CMCAP%2CTCAP%2CMFSUM%2CMFRATIO.MFRATIO2%2CMFRATIO.MFRATIO10%2CSNAME%2CCODE%2CANNOUNMT%2CUVSNEWS&sort=PERCENT&order=desc&count=1000&type=query' \ + -H 'Accept: application/json, text/javascript, */*; q=0.01' \ + -H 'Accept-Language: zh-CN,zh;q=0.9,en;q=0.8' \ + -H 'Connection: keep-alive' \ + -H 'Cookie: _ntes_nnid=07a59ac6cc3c3873093db99e3419a5c7,1652972918736; _ntes_nuid=07a59ac6cc3c3873093db99e3419a5c7; _antanalysis_s_id=1655737843219; s_n_f_l_n3=90474b666b6678eb1655739716131; ne_analysis_trace_id=1655740348110; _ntes_stock_recent_=0601857%7C0601808; _ntes_stock_recent_=0601857%7C0601808; _ntes_stock_recent_=0601857%7C0601808; pgr_n_f_l_n3=90474b666b6678eb165574055174140; vinfo_n_f_l_n3=90474b666b6678eb.1.1.1655737842842.1655738334425.1655740555481' \ + -H 'Referer: http://quotes.money.163.com/old/' \ + -H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.0.0 Safari/537.36' \ + -H 'X-Requested-With: XMLHttpRequest' \ + --compressed \ + --insecure` - // murl := `curl 'http://quotes.money.163.com/hs/service/diyrank.php?host=http%3A%2F%2Fquotes.money.163.com%2Fhs%2Fservice%2Fdiyrank.php&page=1&query=STYPE%3AEQA&fields=NO%2CSYMBOL%2CNAME%2CPRICE%2CPERCENT%2CUPDOWN%2CFIVE_MINUTE%2COPEN%2CYESTCLOSE%2CHIGH%2CLOW%2CVOLUME%2CTURNOVER%2CHS%2CLB%2CWB%2CZF%2CPE%2CMCAP%2CTCAP%2CMFSUM%2CMFRATIO.MFRATIO2%2CMFRATIO.MFRATIO10%2CSNAME%2CCODE%2CANNOUNMT%2CUVSNEWS&sort=PERCENT&order=desc&count=24&type=query' \ - // -H 'Accept: application/json, text/javascript, */*; q=0.01' \ - // -H 'Accept-Language: zh-CN,zh;q=0.9,en;q=0.8' \ - // -H 'Connection: keep-alive' \ - // -H 'Cookie: _ntes_nnid=07a59ac6cc3c3873093db99e3419a5c7,1652972918736; _ntes_nuid=07a59ac6cc3c3873093db99e3419a5c7; _antanalysis_s_id=1655737843219; s_n_f_l_n3=90474b666b6678eb1655739716131; ne_analysis_trace_id=1655740348110; _ntes_stock_recent_=0601857%7C0601808; _ntes_stock_recent_=0601857%7C0601808; _ntes_stock_recent_=0601857%7C0601808; pgr_n_f_l_n3=90474b666b6678eb165574055174140; vinfo_n_f_l_n3=90474b666b6678eb.1.1.1655737842842.1655738334425.1655740555481' \ - // -H 'Referer: http://quotes.money.163.com/old/' \ - // -H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.0.0 Safari/537.36' \ - // -H 'X-Requested-With: XMLHttpRequest' \ - // --compressed \ - // --insecure` + tp := gcurl.Parse(murl).Temporary() + page := tp.QueryParam(`page=\d+`) + var stockCodesFile = "./stock_codes.gob" + var stockCodes []StockCode + f, err := os.Open(stockCodesFile) + if err == nil { + err = gob.NewDecoder(f).Decode(&stockCodes) + if err != nil { + panic(err) + } + } else { + var i int64 = 0 + var pagecount int64 = 100 + for ; i < pagecount; i++ { + page.IntSet(i) + resp, err := tp.Execute() + if err != nil { + panic(err) + } + jr := gjson.ParseBytes(resp.Content()) + pagecount = jr.Get("pagecount").Int() + for _, ljr := range jr.Get("list").Array() { + stockCodes = append(stockCodes, StockCode{CodeStr: ljr.Get("CODE").Str, Code: ljr.Get("SYMBOL").Str}) + } + // log.Println(jr.String()) + } + f, err = os.OpenFile(stockCodesFile, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0664) + if err != nil { + panic(err) + } + err = gob.NewEncoder(f).Encode(&stockCodes) + if err != nil { + panic(err) + } + } - // tp := gcurl.Parse(murl).Temporary() - // page := tp.QueryParam(`page=\d+`) + if f != nil { + err = f.Close() + if err != nil { + panic(err) + } + } - // var i int64 = 0 - // for ; i < 1000; i++ { - // page.IntSet(i) - // resp, err := tp.Execute() - // if err != nil { - // panic(err) - // } + for _, code := range stockCodes { + DownloadDataFromCode(client, &code) + } +} - // log.Println(string(resp.Content())) - // } +func DownloadDataFromCode(client *mongo.Client, code *StockCode) { + // 300731 + durl := `curl 'http://quotes.money.163.com/service/chddata.html?code=${codestr}&start=20171208&end=20220620&fields=TCLOSE;HIGH;LOW;TOPEN;LCLOSE;CHG;PCHG;TURNOVER;VOTURNOVER;VATURNOVER;TCAP;MCAP' \ + -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9' \ + -H 'Accept-Language: zh-CN,zh;q=0.9,en;q=0.8' \ + -H 'Connection: keep-alive' \ + -H 'Cookie: _ntes_nnid=07a59ac6cc3c3873093db99e3419a5c7,1652972918736; _ntes_nuid=07a59ac6cc3c3873093db99e3419a5c7; _antanalysis_s_id=1655737843219; ne_analysis_trace_id=1655740348110; _ntes_stock_recent_=${codestr}%7C0601857%7C0601808; _ntes_stock_recent_=${codestr}%7C0601857%7C0601808; _ntes_stock_recent_=${codestr}%7C0601857%7C0601808; pgr_n_f_l_n3=90474b666b6678eb16557410822304632; vinfo_n_f_l_n3=90474b666b6678eb.1.1.1655737842842.1655738334425.1655741105485' \ + -H 'Referer: http://quotes.money.163.com/trade/lsjysj_${code}.html' \ + -H 'Upgrade-Insecure-Requests: 1' \ + -H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.0.0 Safari/537.36'` - durl := `curl 'http://quotes.money.163.com/service/chddata.html?code=1300731&start=20171208&end=20220620&fields=TCLOSE;HIGH;LOW;TOPEN;LCLOSE;CHG;PCHG;TURNOVER;VOTURNOVER;VATURNOVER;TCAP;MCAP' \ - -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9' \ - -H 'Accept-Language: zh-CN,zh;q=0.9,en;q=0.8' \ - -H 'Connection: keep-alive' \ - -H 'Cookie: _ntes_nnid=07a59ac6cc3c3873093db99e3419a5c7,1652972918736; _ntes_nuid=07a59ac6cc3c3873093db99e3419a5c7; _antanalysis_s_id=1655737843219; ne_analysis_trace_id=1655740348110; _ntes_stock_recent_=1300731%7C0601857%7C0601808; _ntes_stock_recent_=1300731%7C0601857%7C0601808; _ntes_stock_recent_=1300731%7C0601857%7C0601808; pgr_n_f_l_n3=90474b666b6678eb16557410822304632; vinfo_n_f_l_n3=90474b666b6678eb.1.1.1655737842842.1655738334425.1655741105485' \ - -H 'Referer: http://quotes.money.163.com/trade/lsjysj_300731.html' \ - -H 'Upgrade-Insecure-Requests: 1' \ - -H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.0.0 Safari/537.36' \ - --compressed \ - --insecure` + durl = strings.ReplaceAll(durl, `${code}`, code.Code) + durl = strings.ReplaceAll(durl, `${codestr}`, code.CodeStr) resp, err := gcurl.Parse(durl).Temporary().Execute() if err != nil { @@ -93,7 +145,7 @@ func main() { for _, field := range alls[0] { v, _ := GbkToUtf8([]byte(field)) jfield = append(jfield, string(v)) - log.Printf("%#v", string(v)) + // log.Printf("%#v", string(v)) } re, _ := regexp.Compile(`\d+`) @@ -140,8 +192,8 @@ func main() { if err != nil { } - log.Println(r) - + log.Println(code.Code, r) + time.Sleep(time.Second * 2) } func ToFloat(s string) float64 { diff --git a/my.json.7z b/my.json.7z new file mode 100644 index 0000000..f240ba2 Binary files /dev/null and b/my.json.7z differ diff --git a/stock_codes.gob b/stock_codes.gob new file mode 100644 index 0000000..727f08a Binary files /dev/null and b/stock_codes.gob differ