TODO: hook Download
This commit is contained in:
parent
9dafc9d6d9
commit
2c26cae619
4
go.mod
4
go.mod
@ -5,6 +5,7 @@ go 1.18
|
||||
require github.com/tidwall/gjson v1.14.1
|
||||
|
||||
require (
|
||||
github.com/go-rod/rod v0.107.2 // indirect
|
||||
github.com/go-stack/stack v1.8.0 // indirect
|
||||
github.com/golang/snappy v0.0.1 // indirect
|
||||
github.com/klauspost/compress v1.13.6 // indirect
|
||||
@ -13,6 +14,9 @@ require (
|
||||
github.com/xdg-go/scram v1.0.2 // indirect
|
||||
github.com/xdg-go/stringprep v1.0.2 // indirect
|
||||
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
|
||||
github.com/ysmood/goob v0.4.0 // indirect
|
||||
github.com/ysmood/gson v0.7.1 // indirect
|
||||
github.com/ysmood/leakless v0.7.0 // indirect
|
||||
go.mongodb.org/mongo-driver v1.9.1 // indirect
|
||||
golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f // indirect
|
||||
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e // indirect
|
||||
|
||||
10
go.sum
10
go.sum
@ -7,6 +7,8 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
|
||||
github.com/elazarl/goproxy v0.0.0-20210801061803-8e322dfb79c4/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM=
|
||||
github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8=
|
||||
github.com/elazarl/goproxy/ext v0.0.0-20210801061803-8e322dfb79c4/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8=
|
||||
github.com/go-rod/rod v0.107.2 h1:T2nSPKBrHuM/nhGPS/rl9Uwz0dI7T2M9xYF5owqxjMQ=
|
||||
github.com/go-rod/rod v0.107.2/go.mod h1:Au6ufsz7KyXUJVnw6Ljs1nFpsopy+9AJ/lBwGauYBVg=
|
||||
github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
|
||||
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
|
||||
github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
|
||||
@ -40,6 +42,14 @@ github.com/xdg-go/stringprep v1.0.2 h1:6iq84/ryjjeRmMJwxutI51F2GIPlP5BfTvXHeYjyh
|
||||
github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM=
|
||||
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA=
|
||||
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
|
||||
github.com/ysmood/goob v0.4.0 h1:HsxXhyLBeGzWXnqVKtmT9qM7EuVs/XOgkX7T6r1o1AQ=
|
||||
github.com/ysmood/goob v0.4.0/go.mod h1:u6yx7ZhS4Exf2MwciFr6nIM8knHQIE22lFpWHnfql18=
|
||||
github.com/ysmood/got v0.29.5/go.mod h1:pE1l4LOwOBhQg6A/8IAatkGp7uZjnalzrZolnlhhMgY=
|
||||
github.com/ysmood/gotrace v0.6.0/go.mod h1:TzhIG7nHDry5//eYZDYcTzuJLYQIkykJzCRIo4/dzQM=
|
||||
github.com/ysmood/gson v0.7.1 h1:zKL2MTGtynxdBdlZjyGsvEOZ7dkxaY5TH6QhAbTgz0Q=
|
||||
github.com/ysmood/gson v0.7.1/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3RNmg=
|
||||
github.com/ysmood/leakless v0.7.0 h1:XCGdaPExyoreoQd+H5qgxM3ReNbSPFsEXpSKwbXbwQw=
|
||||
github.com/ysmood/leakless v0.7.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ=
|
||||
go.mongodb.org/mongo-driver v1.9.1 h1:m078y9v7sBItkt1aaoe2YlvWEXcD263e1a4E1fBrJ1c=
|
||||
go.mongodb.org/mongo-driver v1.9.1/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
|
||||
150
main.go
150
main.go
@ -5,8 +5,11 @@ import (
|
||||
"context"
|
||||
"encoding/csv"
|
||||
"encoding/gob"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net"
|
||||
"os"
|
||||
"regexp"
|
||||
"strconv"
|
||||
@ -14,6 +17,10 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/474420502/gcurl"
|
||||
"github.com/go-rod/rod"
|
||||
"github.com/go-rod/rod/lib/devices"
|
||||
"github.com/go-rod/rod/lib/launcher"
|
||||
"github.com/go-rod/rod/lib/proto"
|
||||
"github.com/tidwall/gjson"
|
||||
"go.mongodb.org/mongo-driver/mongo"
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
@ -41,9 +48,33 @@ type Stock struct {
|
||||
Code int `json:"股票数字代码" bson:"股票数字代码"`
|
||||
}
|
||||
|
||||
type StockCode struct {
|
||||
CodeStr string // 代地区码
|
||||
Code string // 不带地区码
|
||||
type StockBase struct {
|
||||
// CodeStr string // 代地区码
|
||||
// Code string // 不带地区码
|
||||
|
||||
CODE string `json:"CODE"`
|
||||
FIVE_MINUTE float64 `json:"FIVE_MINUTE"`
|
||||
HIGH float64 `json:"HIGH"`
|
||||
HS float64 `json:"HS"`
|
||||
LB float64 `json:"LB"`
|
||||
LOW float64 `json:"LOW"`
|
||||
MCAP float64 `json:"MCAP"`
|
||||
MFSUM float64 `json:"MFSUM"`
|
||||
NAME string `json:"NAME"`
|
||||
OPEN float64 `json:"OPEN"`
|
||||
PE float64 `json:"PE"`
|
||||
PERCENT float64 `json:"PERCENT"`
|
||||
PRICE float64 `json:"PRICE"`
|
||||
SNAME string `json:"SNAME"`
|
||||
SYMBOL string `json:"SYMBOL"`
|
||||
TCAP float64 `json:"TCAP"`
|
||||
TURNOVER float64 `json:"TURNOVER"`
|
||||
UPDOWN float64 `json:"UPDOWN"`
|
||||
VOLUME float64 `json:"VOLUME"`
|
||||
WB float64 `json:"WB"`
|
||||
YESTCLOSE float64 `json:"YESTCLOSE"`
|
||||
ZF float64 `json:"ZF"`
|
||||
NO float64 `json:"NO"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
@ -52,8 +83,8 @@ func main() {
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
// client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://localhost:27017"))
|
||||
client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://root:6601502@localhost:27017"))
|
||||
client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://localhost:27017"))
|
||||
// client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://root:6601502@localhost:27017"))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
@ -71,7 +102,7 @@ func main() {
|
||||
tp := gcurl.Parse(murl).Temporary()
|
||||
page := tp.QueryParam(`page=\d+`)
|
||||
var stockCodesFile = "./stock_codes.gob"
|
||||
var stockCodes []StockCode
|
||||
var stockCodes []*StockBase
|
||||
f, err := os.Open(stockCodesFile)
|
||||
if err == nil {
|
||||
err = gob.NewDecoder(f).Decode(&stockCodes)
|
||||
@ -87,11 +118,20 @@ func main() {
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
jr := gjson.ParseBytes(resp.Content())
|
||||
pagecount = jr.Get("pagecount").Int()
|
||||
for _, ljr := range jr.Get("list").Array() {
|
||||
stockCodes = append(stockCodes, StockCode{CodeStr: ljr.Get("CODE").Str, Code: ljr.Get("SYMBOL").Str})
|
||||
|
||||
for _, s := range jr.Get("list").Array() {
|
||||
var stockCode StockBase
|
||||
err = json.Unmarshal([]byte(s.String()), &stockCode)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
stockCodes = append(stockCodes, &stockCode)
|
||||
}
|
||||
|
||||
// log.Println(jr.String())
|
||||
}
|
||||
f, err = os.OpenFile(stockCodesFile, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0664)
|
||||
@ -112,23 +152,79 @@ func main() {
|
||||
}
|
||||
|
||||
for _, code := range stockCodes {
|
||||
DownloadDataFromCode(client, &code)
|
||||
if code.MCAP >= 50000000000 {
|
||||
DownloadDataFromCode(client, code)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func DownloadDataFromCode(client *mongo.Client, code *StockCode) {
|
||||
func DownloadDataFromCode(client *mongo.Client, code *StockBase) {
|
||||
// 300731
|
||||
durl := `curl 'http://quotes.money.163.com/service/chddata.html?code=${codestr}&start=20171208&end=20220620&fields=TCLOSE;HIGH;LOW;TOPEN;LCLOSE;CHG;PCHG;TURNOVER;VOTURNOVER;VATURNOVER;TCAP;MCAP' \
|
||||
-H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9' \
|
||||
-H 'Accept-Language: zh-CN,zh;q=0.9,en;q=0.8' \
|
||||
-H 'Connection: keep-alive' \
|
||||
-H 'Cookie: _ntes_nnid=07a59ac6cc3c3873093db99e3419a5c7,1652972918736; _ntes_nuid=07a59ac6cc3c3873093db99e3419a5c7; _antanalysis_s_id=1655737843219; ne_analysis_trace_id=1655740348110; _ntes_stock_recent_=${codestr}%7C0601857%7C0601808; _ntes_stock_recent_=${codestr}%7C0601857%7C0601808; _ntes_stock_recent_=${codestr}%7C0601857%7C0601808; pgr_n_f_l_n3=90474b666b6678eb16557410822304632; vinfo_n_f_l_n3=90474b666b6678eb.1.1.1655737842842.1655738334425.1655741105485' \
|
||||
-H 'Referer: http://quotes.money.163.com/trade/lsjysj_${code}.html' \
|
||||
-H 'Upgrade-Insecure-Requests: 1' \
|
||||
-H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.0.0 Safari/537.36'`
|
||||
durl := `curl 'http://quotes.money.163.com/service/chddata.html?code=${CODE}&start=20170101&end=20220621&fields=TCLOSE;HIGH;LOW;TOPEN;LCLOSE;CHG;PCHG;TURNOVER;VOTURNOVER;VATURNOVER;TCAP;MCAP' \
|
||||
-H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9' \
|
||||
-H 'Accept-Language: zh-CN,zh;q=0.9,en;q=0.8' \
|
||||
-H 'Connection: keep-alive' \
|
||||
-H 'Cookie: _ntes_nnid=07a59ac6cc3c3873093db99e3419a5c7,1652972918736; _ntes_nuid=07a59ac6cc3c3873093db99e3419a5c7; _antanalysis_s_id=1655737843219; ne_analysis_trace_id=1655740348110; _ntes_stock_recent_=${CODE}%7C1300660%7C1300731%7C0601857%7C0601808; _ntes_stock_recent_=${CODE}%7C1300660%7C1300731%7C0601857%7C0601808; _ntes_stock_recent_=${CODE}%7C1300660%7C1300731%7C0601857%7C0601808; s_n_f_l_n3=90474b666b6678eb1655828892281; pgr_n_f_l_n3=90474b666b6678eb16558271774556486; vinfo_n_f_l_n3=90474b666b6678eb.1.6.1655737842842.1655828160869.1655828914287' \
|
||||
-H 'Referer: http://quotes.money.163.com/trade/lsjysj_${SYMBOL}.html' \
|
||||
-H 'Upgrade-Insecure-Requests: 1' \
|
||||
-H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.0.0 Safari/537.36' `
|
||||
|
||||
durl = strings.ReplaceAll(durl, `${code}`, code.Code)
|
||||
durl = strings.ReplaceAll(durl, `${codestr}`, code.CodeStr)
|
||||
// http://quotes.money.163.com/0601988.html
|
||||
|
||||
screen := devices.Device{
|
||||
Title: "Laptop with MDPI screen",
|
||||
Capabilities: []string{"touch", "mobile"},
|
||||
UserAgent: "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36",
|
||||
Screen: devices.Screen{
|
||||
DevicePixelRatio: 1,
|
||||
Horizontal: devices.ScreenSize{
|
||||
Width: 1920,
|
||||
Height: 1080,
|
||||
},
|
||||
},
|
||||
}
|
||||
port := 40000
|
||||
log.Println("get port:", port)
|
||||
rodlauncher := launcher.New().
|
||||
Bin(`google-chrome`).
|
||||
RemoteDebuggingPort(port).
|
||||
Set("user-data-dir", fmt.Sprintf("/tmp/%s_rod", "money-money")).
|
||||
Delete("headless")
|
||||
//debug url
|
||||
launchers := rodlauncher.MustLaunch()
|
||||
fmt.Printf("debug url: %s\n", launchers)
|
||||
//连接浏览器
|
||||
browser := rod.New().ControlURL(launchers).MustConnect()
|
||||
page := browser.DefaultDevice(screen).MustPage(fmt.Sprintf("http://quotes.money.163.com/%s.html", code.CODE))
|
||||
|
||||
p := page.Timeout(time.Second * 5)
|
||||
ele, err := p.ElementsX("//a[@href='/trade/lsjysj_600096.html#01b07']/@href")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
time.Sleep(time.Second * 3)
|
||||
|
||||
if iter := ele.First(); iter != nil {
|
||||
|
||||
urlpath, err := iter.HTML()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
log.Println(page.MustInfo().URL)
|
||||
page.MustNavigate("http://quotes.money.163.com" + urlpath)
|
||||
|
||||
ahref := page.MustElementX("//a[@id='downloadData']")
|
||||
|
||||
ahref.Click(proto.InputMouseButtonLeft)
|
||||
time.Sleep(time.Second)
|
||||
e := page.MustElementX("//a[@class='blue_btn submit']")
|
||||
e.Click(proto.InputMouseButtonLeft)
|
||||
|
||||
}
|
||||
|
||||
durl = strings.ReplaceAll(durl, `${SYMBOL}`, code.SYMBOL)
|
||||
durl = strings.ReplaceAll(durl, `${CODE}`, code.CODE)
|
||||
|
||||
resp, err := gcurl.Parse(durl).Temporary().Execute()
|
||||
if err != nil {
|
||||
@ -192,7 +288,7 @@ func DownloadDataFromCode(client *mongo.Client, code *StockCode) {
|
||||
if err != nil {
|
||||
|
||||
}
|
||||
log.Println(code.Code, r)
|
||||
log.Println(code.CODE, r)
|
||||
time.Sleep(time.Second * 2)
|
||||
}
|
||||
|
||||
@ -215,3 +311,15 @@ func GbkToUtf8(s []byte) ([]byte, error) {
|
||||
}
|
||||
return d, nil
|
||||
}
|
||||
|
||||
func GetPort() int {
|
||||
l, _ := net.Listen("tcp", ":0") // listen on localhost
|
||||
port := l.Addr().(*net.TCPAddr).Port
|
||||
err := l.Close()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
// ip := l.Addr().(*net.TCPAddr).IP
|
||||
return port
|
||||
|
||||
}
|
||||
|
||||
BIN
stock_codes.gob
BIN
stock_codes.gob
Binary file not shown.
Loading…
x
Reference in New Issue
Block a user