diff --git a/logdb.yaml b/logdb.yaml new file mode 100644 index 0000000..76f3ee9 --- /dev/null +++ b/logdb.yaml @@ -0,0 +1,6 @@ +charset: utf8mb4 +db: test_log +hosts: [192.168.6.101,192.168.6.102,192.168.6.103,192.168.6.104,192.168.6.105] +password: ag-spider-log +port: 4000 +user: spider \ No newline at end of file diff --git a/parser.go b/parser.go index cca4d1f..5a6b811 100644 --- a/parser.go +++ b/parser.go @@ -1,5 +1,11 @@ package parser +import ( + "log" + + "474420502.top/test/logdb" +) + type Resource struct { Rawurl string `json:"url"` // 当前采集的素材资源 url 是 Type string `json:"type"` @@ -95,9 +101,44 @@ type ADParser struct { DeviceInfo DeviceInfo `json:"deviceInfo"` SectionInfo SectionInfo `json:"sectionInfo,omitempty"` + + IParser } -func NewADParser() *ADParser { +// IParser 要实现的解析接口 +type IParser interface { + Parser(adstring string) (string, error) + GetSpiderID() int +} + +type Toutiao struct { +} + +func ADParserServer(adp IParser) { + db := logdb.New("logdb.yaml") + tt := &Toutiao{} + ad := NewADParser(12) + + adresponse := db.ADParserSelect(adp.GetSpiderID()) // select from db + for _, adr := range adresponse { + ParserAndSendMQ(&adr, adp) + } +} + +func ParserAndSendMQ(adr *logdb.ADResonse, adp IParser) error { + + if pjson, err := adp.Parser(adr.Response); err != nil { + log.Println(err) + return err + } else { + // send pjson to mq + // update UID status finish + } + + return nil +} + +func NewADParser(SpiderID int) *ADParser { adp := &ADParser{} adp.Resources = make([]Resource, 0) adp.Format = make([]int, 0)