From 3f5adf1ba94a8ff41518d79e3d4d05be8bcaae9b Mon Sep 17 00:00:00 2001 From: Josh Baker Date: Sun, 7 May 2017 18:26:54 -0700 Subject: [PATCH] New gjson.Unmarshal function It's a drop in replacement for json.Unmarshal and you can typically see a 3 to 4 times boost in performance without the need for external tools or generators. This function works almost identically to json.Unmarshal except that it expects the json to be well-formed prior to being called. Invalid json will not panic, but it may return back unexpected results. Therefore the return value of this function will always be nil. Another difference is that gjson.Unmarshal will automatically attempt to convert JSON values to any Go type. For example, the JSON string "100" or the JSON number 100 can be equally assigned to Go string, int, byte, uint64, etc. This rule applies to all types. --- README.md | 54 ++++++++++++++++++- gjson.go | 121 ++++++++++++++++++++++++++++++++++++++++++ gjson_test.go | 142 ++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 316 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5832c54..4d5a53e 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@

get a json value quickly

GJSON is a Go package that provides a [fast](#performance) and [simple](#get-a-value) way to get values from a json document. -It has features such as [one line retrieval](#get-a-value), [dot notation paths](#path-syntax), [iteration](#iterate-through-an-object-or-array), and [map unmarshalling](#unmarshal-to-a-map). +It has features such as [one line retrieval](#get-a-value), [dot notation paths](#path-syntax), [iteration](#iterate-through-an-object-or-array). It can also [unmarshal](#unmarshalling) 3 to 4 times faster than the standard Go `json/encoding` unmarshaller. Getting Started =============== @@ -233,6 +233,58 @@ if gjson.Get(json, "name.last").Exists(){ } ``` +## Unmarshalling + +There's a `gjson.Unmarshal` function that loads json data into the value. +It's a drop in replacement for `json.Unmarshal` and you can typically see a +3 to 4 times boost in performance without the need for external tools or +generators. + +This function works almost identically to `json.Unmarshal` except that it +expects the json to be well-formed prior to being called. Invalid json +will not panic, but it may return back unexpected results. Therefore the +return value of this function will always be nil. + +Another difference is that `gjson.Unmarshal` will automatically attempt to +convert JSON values to any Go type. For example, the JSON string "100" or +the JSON number 100 can be equally assigned to Go string, int, byte, uint64, +etc. This rule applies to all types. + + +```go +package main + +import ( + "fmt" + + "github.com/tidwall/gjson" +) + +type Animal struct { + Type string `json:"type"` + Sound string `json:"sound"` + Age int `json:"age"` +} + +var json = `{ + "type": "Dog", + "Sound": "Bark", + "Age": "11" +}` + +func main() { + var dog Animal + gjson.Unmarshal([]byte(json), &dog) + fmt.Printf("type: %s, sound: %s, age: %d\n", dog.Type, dog.Sound, dog.Age) +} +``` + +This will print: + +``` +type: Dog, sound: Bark, age: 11 +``` + ## Unmarshal to a map To unmarshal to a `map[string]interface{}`: diff --git a/gjson.go b/gjson.go index 82bd957..77ed4c6 100644 --- a/gjson.go +++ b/gjson.go @@ -2,8 +2,12 @@ package gjson import ( + "encoding/base64" + "encoding/json" "reflect" "strconv" + "strings" + "sync" "time" "unicode/utf16" "unicode/utf8" @@ -1954,3 +1958,120 @@ func getMany512(json string, i int, paths []string) ([]Result, bool) { _, ok := parseGetMany(json, i, 0, 0, paths, completed, matches, results) return results, ok } + +var fieldsmu sync.RWMutex +var fields = make(map[string]map[string]int) + +func assign(jsval Result, goval reflect.Value) { + if jsval.Type == Null { + return + } + switch goval.Kind() { + default: + case reflect.Ptr: + if !goval.IsNil() { + newval := reflect.New(goval.Elem().Type()) + assign(jsval, newval.Elem()) + goval.Elem().Set(newval.Elem()) + } else { + newval := reflect.New(goval.Type().Elem()) + assign(jsval, newval.Elem()) + goval.Set(newval) + } + case reflect.Struct: + fieldsmu.RLock() + sf := fields[goval.Type().String()] + fieldsmu.RUnlock() + if sf == nil { + fieldsmu.Lock() + sf = make(map[string]int) + for i := 0; i < goval.Type().NumField(); i++ { + f := goval.Type().Field(i) + tag := strings.Split(f.Tag.Get("json"), ",")[0] + if tag != "-" { + if tag != "" { + sf[tag] = i + sf[f.Name] = i + } else { + sf[f.Name] = i + } + } + } + fields[goval.Type().String()] = sf + fieldsmu.Unlock() + } + jsval.ForEach(func(key, value Result) bool { + if idx, ok := sf[key.Str]; ok { + f := goval.Field(idx) + if f.CanSet() { + assign(value, f) + } + } + return true + }) + case reflect.Slice: + if goval.Type().Elem().Kind() == reflect.Uint8 && jsval.Type == String { + data, _ := base64.StdEncoding.DecodeString(jsval.String()) + goval.Set(reflect.ValueOf(data)) + } else { + jsvals := jsval.Array() + slice := reflect.MakeSlice(goval.Type(), len(jsvals), len(jsvals)) + for i := 0; i < len(jsvals); i++ { + assign(jsvals[i], slice.Index(i)) + } + goval.Set(slice) + } + case reflect.Array: + i, n := 0, goval.Len() + jsval.ForEach(func(_, value Result) bool { + if i == n { + return false + } + assign(value, goval.Index(i)) + i++ + return true + }) + case reflect.Map: + if goval.Type().Key().Kind() == reflect.String && goval.Type().Elem().Kind() == reflect.Interface { + goval.Set(reflect.ValueOf(jsval.Value())) + } + case reflect.Interface: + goval.Set(reflect.ValueOf(jsval.Value())) + case reflect.Bool: + goval.SetBool(jsval.Bool()) + case reflect.Float32, reflect.Float64: + goval.SetFloat(jsval.Float()) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + goval.SetInt(jsval.Int()) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + goval.SetUint(jsval.Uint()) + case reflect.String: + goval.SetString(jsval.String()) + } + if len(goval.Type().PkgPath()) > 0 { + v := goval.Addr() + if v.Type().NumMethod() > 0 { + if u, ok := v.Interface().(json.Unmarshaler); ok { + u.UnmarshalJSON([]byte(jsval.Raw)) + } + } + } +} + +// Unmarshal loads the JSON data into the value pointed to by v. +// +// This function works almost identically to json.Unmarshal except that it +// expects that the json is well-formed prior to being called. Invalid json +// will not panic, but it may return back unexpected results. Therefore the +// return value of this function will always be nil. +// +// Another difference is that gjson.Unmarshal will automatically attempt to +// convert JSON values to any Go type. For example, the JSON string "100" or +// the JSON number 100 can be equally assigned to Go string, int, byte, uint64, +// etc. This rule applies to all types. +func Unmarshal(data []byte, v interface{}) error { + if v := reflect.ValueOf(v); v.Kind() == reflect.Ptr { + assign(ParseBytes(data), v) + } + return nil +} diff --git a/gjson_test.go b/gjson_test.go index 137974f..235a310 100644 --- a/gjson_test.go +++ b/gjson_test.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "math/rand" + "reflect" "strings" "testing" "time" @@ -787,6 +788,119 @@ func TestRandomMany(t *testing.T) { } } +type ComplicatedType struct { + unsettable int + Tagged string `json:"tagged"` + NotTagged bool + Nested struct { + Yellow string `json:"yellow"` + } + NestedTagged struct { + Green string + Map map[string]interface{} + Ints struct { + Int int `json:"int"` + Int8 int8 + Int16 int16 + Int32 int32 + Int64 int64 `json:"int64"` + } + Uints struct { + Uint uint + Uint8 uint8 + Uint16 uint16 + Uint32 uint32 + Uint64 uint64 + } + Floats struct { + Float64 float64 + Float32 float32 + } + Byte byte + Bool bool + } `json:"nestedTagged"` + LeftOut string `json:"-"` + SelfPtr *ComplicatedType + SelfSlice []ComplicatedType + SelfSlicePtr []*ComplicatedType + SelfPtrSlice *[]ComplicatedType + Interface interface{} `json:"interface"` + Array [3]int + Time time.Time `json:"time"` + Binary []byte + NonBinary []byte +} + +var complicatedJSON = ` +{ + "tagged": "OK", + "Tagged": "KO", + "NotTagged": true, + "unsettable": 101, + "Nested": { + "Yellow": "Green", + "yellow": "yellow" + }, + "nestedTagged": { + "Green": "Green", + "Map": { + "this": "that", + "and": "the other thing" + }, + "Ints": { + "Uint": 99, + "Uint16": 16, + "Uint32": 32, + "Uint64": 65 + }, + "Uints": { + "int": -99, + "Int": -98, + "Int16": -16, + "Int32": -32, + "int64": -64, + "Int64": -65 + }, + "Uints": { + "Float32": 32.32, + "Float64": 64.64 + }, + "Byte": 254, + "Bool": true + }, + "LeftOut": "you shouldn't be here", + "SelfPtr": {"tagged":"OK","nestedTagged":{"Ints":{"Uint32":32}}}, + "SelfSlice": [{"tagged":"OK","nestedTagged":{"Ints":{"Uint32":32}}}], + "SelfSlicePtr": [{"tagged":"OK","nestedTagged":{"Ints":{"Uint32":32}}}], + "SelfPtrSlice": [{"tagged":"OK","nestedTagged":{"Ints":{"Uint32":32}}}], + "interface": "Tile38 Rocks!", + "Interface": "Please Download", + "Array": [0,2,3,4,5], + "time": "2017-05-07T13:24:43-07:00", + "Binary": "R0lGODlhPQBEAPeo", + "NonBinary": [9,3,100,115] +} +` + +func TestUnmarshal(t *testing.T) { + var s1 ComplicatedType + var s2 ComplicatedType + if err := json.Unmarshal([]byte(complicatedJSON), &s1); err != nil { + t.Fatal(err) + } + if err := Unmarshal([]byte(complicatedJSON), &s2); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(&s1, &s2) { + t.Fatal("not equal") + } + var str string + if err := json.Unmarshal([]byte(Get(complicatedJSON, "LeftOut").Raw), &str); err != nil { + t.Fatal(err) + } + assert(t, str == Get(complicatedJSON, "LeftOut").String()) +} + type BenchStruct struct { Widget struct { Window struct { @@ -900,6 +1014,34 @@ func BenchmarkGJSONUnmarshalMap(t *testing.B) { t.N *= len(benchPaths) // because we are running against 3 paths } +func BenchmarkGJSONUnmarshalStruct(t *testing.B) { + t.ReportAllocs() + t.ResetTimer() + for i := 0; i < t.N; i++ { + for j := 0; j < len(benchPaths); j++ { + var s BenchStruct + if err := Unmarshal([]byte(exampleJSON), &s); err != nil { + t.Fatal(err) + } + switch benchPaths[j] { + case "widget.window.name": + if s.Widget.Window.Name == "" { + t.Fatal("did not find the value") + } + case "widget.image.hOffset": + if s.Widget.Image.HOffset == 0 { + t.Fatal("did not find the value") + } + case "widget.text.onMouseUp": + if s.Widget.Text.OnMouseUp == "" { + t.Fatal("did not find the value") + } + } + } + } + t.N *= len(benchPaths) // because we are running against 3 paths +} + func BenchmarkJSONUnmarshalMap(t *testing.B) { t.ReportAllocs() t.ResetTimer()