diff --git a/parser.go b/parser.go index 705eef6..7b0f41b 100644 --- a/parser.go +++ b/parser.go @@ -4,8 +4,11 @@ import ( "encoding/json" "io/ioutil" "log" + "strconv" + "strings" "sync" + "github.com/tidwall/gjson" "gopkg.in/yaml.v2" "474420502.top/test/logdb" @@ -208,6 +211,21 @@ func parserAndSendMQ(adp IParser, adrChan chan logdb.ADResonse, wg *sync.WaitGro } } +// IsJSON 测试数据是否为JSON +func IsJSON(data string) bool { + var v interface{} + if gjson.Unmarshal([]byte(data), &v) != nil { + data = strings.Trim(data, "\"") + unq, _ := strconv.Unquote("\"" + data + "\"") + data = unq + if gjson.Unmarshal([]byte(data), &v) != nil { + // return "", errors.New("json invalid") + return false + } + } + return true +} + // NewADParser 创建一个ADParser的类, 包含很多传到终端的所有结构 func NewADParser(SpiderID int) *ADParser { adp := &ADParser{} diff --git a/parser_test.go b/parser_test.go index a8a4871..4d0ec0d 100644 --- a/parser_test.go +++ b/parser_test.go @@ -2,10 +2,9 @@ package parser import ( "encoding/json" + "errors" "log" "regexp" - "strconv" - "strings" "testing" "github.com/tidwall/gjson" @@ -33,7 +32,9 @@ func (tt *Toutiao) ToDoParser(adstring string) (presult string, err error) { defer func() { if ierr := recover(); ierr != nil { - log.Println(ierr) + log.Println(presult, ierr) + err = ierr.(error) + presult = "" } }() @@ -43,26 +44,21 @@ func (tt *Toutiao) ToDoParser(adstring string) (presult string, err error) { } for _, data := range adlist { - label := data[0:10] - log.Println("test json -----------------------") - var v interface{} - log.Println(label, gjson.Unmarshal([]byte(data), &v), gjson.Parse(data).IsObject()) - - data = strings.Trim(data, "\"") - unq, _ := strconv.Unquote("\"" + data + "\"") - log.Println(label, gjson.Unmarshal([]byte(data), &v), gjson.Parse(data).IsObject()) - data = unq - log.Println(label, gjson.Unmarshal([]byte(data), &v), gjson.Parse(data).IsObject()) + if IsJSON(data) { + return "", errors.New("json invalid") + } result := gjson.Parse(data) // log.Println(result.String()) - if result.Get("mixed").Exists() { - log.Println("this is mixed") - log.Println(result.Get("mixed.track_url").Exists()) + + for i, rkey := range []string{"app", "mixed", "image_recom"} { + root := result.Get("mixed") + if root.Exists() { + + } } - json.Unmarshal([]byte(data), &v) // spew.Dump(v) regexp.Compile(``) }