doc/money-money
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

最新完美算法

Browse Source
This commit is contained in:
eson 2022-06-23 17:06:00 +08:00
parent 4c5abd04e4
commit 58c19218bf
4 changed files with 192 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
api.go Normal file
View File

@ -0,0 +1,28 @@
package moneymoney
import (
"github.com/474420502/gcurl"
)
func GetFromAPI() {
tp := gcurl.Parse(`curl 'http://quotes.money.163.com/service/chddata.html?code=0601398&start=20140720&end=20150508'`).Temporary()
pcode := tp.QueryParam("code")
start := tp.QueryParam("start")
end := tp.QueryParam("end")
for _, code := range GetStocks() {
if code.MCAP >= 20000000000 {
pcode.StringSet(code.CODE)
start.StringSet("20170101")
end.StringSet("20220622")
resp, err := tp.Execute()
if err != nil {
panic(err)
}
SaveFromCSV(resp.Content())
}
}
}

243
main.go
View File

@ -29,23 +29,23 @@ import (
)
type Stock struct {
Date string `json:"Date" bson:"Date"` // 日期
CodeStr string `json:"CodeStr" bson:"CodeStr"` // 股票代码
Name string `json:"Name" bson:"Name"` // 名称
ClosingPrice float64 `json:"ClosingPrice" bson:"ClosingPrice"` // 收盘价
MaxPrice float64 `json:"MaxPrice" bson:"MaxPrice"` // 最高价
MinPrice float64 `json:"MinPrice" bson:"MinPrice"` // 最低价
OpeningPrice float64 `json:"OpeningPrice" bson:"OpeningPrice"` // 开盘价
PreviousClosingPrice float64 `json:"PreviousClosingPrice" bson:"PreviousClosingPrice"` // 前收盘
UpsDowns float64 `json:"UpsDowns" bson:"UpsDowns"` // 涨跌额
UpsDownsRatio float64 `json:"UpsDownsRatio" bson:"UpsDownsRatio"` // 涨跌幅
TurnoverRate float64 `json:"TurnoverRate" bson:"TurnoverRate"` // 换手率
Volume float64 `json:"Volume" bson:"Volume"` // 成交量
Turnover float64 `json:"Turnover" bson:"Turnover"` // 成交金额
MarketValue float64 `json:"MarketValue" bson:"MarketValue"` // 总市值
CirculatingMarketValue float64 `json:"CirculatingMarketValue" bson:"CirculatingMarketValue"` // 流通市值
Date string `json:"日期" bson:"日期"`
CodeStr string `json:"股票代码" bson:"股票代码"`
Name string `json:"名称" bson:"名称"`
ClosingPrice float64 `json:"收盘价" bson:"收盘价"`
MaxPrice float64 `json:"最高价" bson:"最高价"`
MinPrice float64 `json:"最低价" bson:"最低价"`
OpeningPrice float64 `json:"开盘价" bson:"开盘价"`
PreviousClosingPrice float64 `json:"前收盘" bson:"前收盘"`
UpsDowns float64 `json:"涨跌额" bson:"涨跌额"`
UpsDownsRatio float64 `json:"涨跌幅" bson:"涨跌幅"`
TurnoverRate float64 `json:"换手率" bson:"换手率"`
Volume float64 `json:"成交量" bson:"成交量"`
Turnover float64 `json:"成交金额" bson:"成交金额"`
MarketValue float64 `json:"总市值" bson:"总市值"`
CirculatingMarketValue float64 `json:"流通市值" bson:"流通市值"`
Code string `json:"Code" bson:"Code"` // 股票数字代码
Code int64 `json:"股票数字代码" bson:"股票数字代码"`
}
type StockBase struct {
@ -117,49 +117,52 @@ func GetDefaultPage() *rod.Page {
return DefaultPage
}
func main() {
var client *mongo.Client
var err error
func init() {
log.SetFlags(log.Llongfile | log.LstdFlags)
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
client, err = mongo.Connect(context.TODO(), options.Client().ApplyURI("mongodb://root:6601502@localhost:27017"))
if err != nil {
panic(err)
}
}
func main() {
for _, code := range GetStocks() {
if code.MCAP >= 50000000000 {
DownloadDataFromCode(code)
}
}
}
func GetStocks() []*StockBase {
// client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://localhost:27017"))
client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://root:6601502@localhost:27017"))
cur, err := client.Database("money").Collection("stock").Distinct(context.TODO(), "股票数字代码", bson.M{})
if err != nil {
panic(err)
}
cur, err := client.Database("money").Collection("stock").Aggregate(context.TODO(), bson.A{
bson.M{"$group": bson.M{"_id": "$股票数字代码"}},
})
if err != nil {
panic(err)
}
var skipMap map[int64]bool = make(map[int64]bool)
var skipMap map[string]bool = make(map[string]bool)
for cur.Next(context.TODO()) {
var doc bson.M
err = cur.Decode(&doc)
for _, idoc := range cur {
var scode = idoc.(int64)
// err = cur.Decode(&doc)
if err == nil {
skipMap[strconv.Itoa(int(doc["_id"].(int32)))] = true
skipMap[scode] = true
} else {
log.Panic(err)
}
}
murl := `curl 'http://quotes.money.163.com/hs/service/diyrank.php?host=http%3A%2F%2Fquotes.money.163.com%2Fhs%2Fservice%2Fdiyrank.php&page=1&query=STYPE%3AEQA&fields=NO%2CSYMBOL%2CNAME%2CPRICE%2CPERCENT%2CUPDOWN%2CFIVE_MINUTE%2COPEN%2CYESTCLOSE%2CHIGH%2CLOW%2CVOLUME%2CTURNOVER%2CHS%2CLB%2CWB%2CZF%2CPE%2CMCAP%2CTCAP%2CMFSUM%2CMFRATIO.MFRATIO2%2CMFRATIO.MFRATIO10%2CSNAME%2CCODE%2CANNOUNMT%2CUVSNEWS&sort=PERCENT&order=desc&count=1000&type=query' \
-H 'Accept: application/json, text/javascript, */*; q=0.01' \
-H 'Accept-Language: zh-CN,zh;q=0.9,en;q=0.8' \
-H 'Connection: keep-alive' \
-H 'Cookie: _ntes_nnid=07a59ac6cc3c3873093db99e3419a5c7,1652972918736; _ntes_nuid=07a59ac6cc3c3873093db99e3419a5c7; _antanalysis_s_id=1655737843219; s_n_f_l_n3=90474b666b6678eb1655739716131; ne_analysis_trace_id=1655740348110; _ntes_stock_recent_=0601857%7C0601808; _ntes_stock_recent_=0601857%7C0601808; _ntes_stock_recent_=0601857%7C0601808; pgr_n_f_l_n3=90474b666b6678eb165574055174140; vinfo_n_f_l_n3=90474b666b6678eb.1.1.1655737842842.1655738334425.1655740555481' \
-H 'Referer: http://quotes.money.163.com/old/' \
-H 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.0.0 Safari/537.36' \
-H 'X-Requested-With: XMLHttpRequest'`
tp := gcurl.Parse(murl).Temporary()
page := tp.QueryParam(`page=\d+`)
var stockCodesFile = "./stock_codes.gob"
var stockCodes []*StockBase
f, err := os.Open(stockCodesFile)
@ -169,38 +172,34 @@ func main() {
panic(err)
}
} else {
var i int64 = 0
var pagecount int64 = 100
for ; i < pagecount; i++ {
page.IntSet(i)
resp, err := tp.Execute()
murl := `curl 'http://quotes.money.163.com/hs/service/diyrank.php?count=5000'`
tp := gcurl.Parse(murl).Temporary()
resp, err := tp.Execute()
if err != nil {
panic(err)
}
jr := gjson.ParseBytes(resp.Content())
for _, s := range jr.Get("list").Array() {
var stockCode StockBase
err = json.Unmarshal([]byte(s.String()), &stockCode)
if err != nil {
panic(err)
}
jr := gjson.ParseBytes(resp.Content())
pagecount = jr.Get("pagecount").Int()
for _, s := range jr.Get("list").Array() {
var stockCode StockBase
err = json.Unmarshal([]byte(s.String()), &stockCode)
if err != nil {
panic(err)
}
stockCodes = append(stockCodes, &stockCode)
}
// log.Println(jr.String())
}
f, err = os.OpenFile(stockCodesFile, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0664)
if err != nil {
panic(err)
}
err = gob.NewEncoder(f).Encode(&stockCodes)
if err != nil {
panic(err)
stockCodes = append(stockCodes, &stockCode)
}
// log.Println(jr.String())
}
f, err = os.OpenFile(stockCodesFile, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0664)
if err != nil {
panic(err)
}
err = gob.NewEncoder(f).Encode(&stockCodes)
if err != nil {
panic(err)
}
if f != nil {
@ -210,18 +209,30 @@ func main() {
}
}
re, _ := regexp.Compile(`\d+`)
log.Println("stocks", len(stockCodes))
var result []*StockBase
for _, code := range stockCodes {
if _, ok := skipMap[code.SYMBOL]; ok {
scode, err := strconv.ParseInt(re.FindString(code.SYMBOL), 10, 64)
if err != nil {
panic(err)
}
if _, ok := skipMap[scode]; ok {
continue
}
if code.MCAP >= 50000000000 {
DownloadDataFromCode(client, code)
}
result = append(result, code)
// if code.MCAP >= 50000000000 {
// DownloadDataFromCode(client, code)
// }
}
return result
}
func DownloadDataFromCode(client *mongo.Client, code *StockBase) {
func DownloadDataFromCode(code *StockBase) {
// 300731
// durl := `curl 'http://quotes.money.163.com/service/chddata.html?code=${CODE}&start=20170101&end=20220621&fields=TCLOSE;HIGH;LOW;TOPEN;LCLOSE;CHG;PCHG;TURNOVER;VOTURNOVER;VATURNOVER;TCAP;MCAP' \
// -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9' \
@ -234,8 +245,6 @@ func DownloadDataFromCode(client *mongo.Client, code *StockBase) {
// http://quotes.money.163.com/0601988.html
cstock := client.Database("money").Collection("stock")
page := GetDefaultPage()
stockurl := fmt.Sprintf("http://quotes.money.163.com/%s.html", code.CODE)
log.Println(stockurl)
@ -298,7 +307,7 @@ func DownloadDataFromCode(client *mongo.Client, code *StockBase) {
re, _ := regexp.Compile(`\d+`)
// var stocks []mongo.WriteModel
var stocks []mongo.WriteModel
for _, line := range alls[1:] {
var fields []string
@ -308,9 +317,12 @@ func DownloadDataFromCode(client *mongo.Client, code *StockBase) {
}
code := re.FindString(fields[1])
code, err := strconv.ParseInt(re.FindString(fields[1]), 10, 64)
if err != nil {
panic(err)
}
s := Stock{
s := &Stock{
Date: fields[0],
CodeStr: fields[1],
Name: fields[2],
@ -329,15 +341,80 @@ func DownloadDataFromCode(client *mongo.Client, code *StockBase) {
Code: code,
}
cstock.InsertOne(context.TODO(), mongo.InsertOneModel{Document: s})
stocks = append(stocks, &mongo.InsertOneModel{Document: s})
}
log.Println(code.SYMBOL)
cstock := client.Database("money").Collection("stock")
r, err := cstock.BulkWrite(context.TODO(), stocks)
if err != nil {
log.Println(err)
}
log.Println(code.SYMBOL, r)
time.Sleep(time.Second * 1)
}
}
func SaveFromCSV(downloaddata []byte) {
reader := csv.NewReader(bytes.NewBuffer(downloaddata))
alls, err := reader.ReadAll()
if err != nil {
panic(err)
}
var jfield []string
for _, field := range alls[0] {
v, _ := GbkToUtf8([]byte(field))
jfield = append(jfield, string(v))
// log.Printf("%#v", string(v))
}
re, _ := regexp.Compile(`\d+`)
var stocks []mongo.WriteModel
for _, line := range alls[1:] {
var fields []string
for _, field := range line {
v, _ := GbkToUtf8([]byte(field))
fields = append(fields, string(v))
}
code, err := strconv.ParseInt(re.FindString(fields[1]), 10, 64)
if err != nil {
panic(err)
}
s := &Stock{
Date: fields[0],
CodeStr: fields[1],
Name: fields[2],
ClosingPrice: ToFloat(fields[3]),
MaxPrice: ToFloat(fields[4]),
MinPrice: ToFloat(fields[5]),
OpeningPrice: ToFloat(fields[6]),
PreviousClosingPrice: ToFloat(fields[7]),
UpsDowns: ToFloat(fields[8]),
UpsDownsRatio: ToFloat(fields[9]),
TurnoverRate: ToFloat(fields[10]),
Volume: ToFloat(fields[11]),
Turnover: ToFloat(fields[12]),
MarketValue: ToFloat(fields[13]),
CirculatingMarketValue: ToFloat(fields[14]),
Code: code,
}
stocks = append(stocks, &mongo.InsertOneModel{Document: s})
}
cstock := client.Database("money").Collection("stock")
r, err := cstock.BulkWrite(context.TODO(), stocks)
if err != nil {
log.Println(err)
}
log.Println(r)
}
func ToFloat(s string) float64 {
if s == "None" {
return 0

4
main_test.go
View File

@ -2,6 +2,10 @@ package moneymoney
import "testing"
func TestCase2(t *testing.T) {
GetFromAPI()
}
func TestCase1(t *testing.T) {
main()
}

BIN
stock_codes.gob
View File

Binary file not shown.