From 0360deb6d803e8c271363ce5f6c85d6cd843a3a0 Mon Sep 17 00:00:00 2001 From: tidwall Date: Mon, 10 Feb 2020 11:13:30 -0700 Subject: [PATCH] Added new modifiers `@flatten` Flattens an array with child arrays. [1,[2],[3,4],[5,[6,7]]] -> [1,2,3,4,5,[6,7]] The {"deep":true} arg can be provide for deep flattening. [1,[2],[3,4],[5,[6,7]]] -> [1,2,3,4,5,6,7] The original json is returned when the json is not an array. `@join` Joins multiple objects into a single object. [{"first":"Tom"},{"last":"Smith"}] -> {"first","Tom","last":"Smith"} The arg can be "true" to specify that duplicate keys should be preserved. [{"first":"Tom","age":37},{"age":41}] -> {"first","Tom","age":37,"age":41} Without preserved keys: [{"first":"Tom","age":37},{"age":41}] -> {"first","Tom","age":41} The original json is returned when the json is not an object. `@valid` Ensures that the json is valid before moving on. An empty string is returned when the json is not valid, otherwise it returns the original json. --- README.md | 6 ++- SYNTAX.md | 7 ++- gjson.go | 131 +++++++++++++++++++++++++++++++++++++++++++++++ gjson_test.go | 137 ++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 278 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index cab0f9f..4108deb 100644 --- a/README.md +++ b/README.md @@ -193,11 +193,15 @@ we'll get `children` array and reverse the order: "children|@reverse|0" >> "Jack" ``` -There are currently three built-in modifiers: +There are currently the following built-in modifiers: - `@reverse`: Reverse an array or the members of an object. - `@ugly`: Remove all whitespace from a json document. - `@pretty`: Make the json document more human readable. +- `@this`: Returns the current element. It can be used to retrieve the root element. +- `@valid`: Ensure the json document is valid. +- `@flatten`: Flattens an array. +- `@join`: Joins multiple objects into a single object. ### Modifier arguments diff --git a/SYNTAX.md b/SYNTAX.md index ed584de..9558019 100644 --- a/SYNTAX.md +++ b/SYNTAX.md @@ -181,12 +181,15 @@ children.@reverse ["Jack","Alex","Sara"] children.@reverse.0 "Jack" ``` -There are currently three built-in modifiers: +There are currently the following built-in modifiers: - `@reverse`: Reverse an array or the members of an object. - `@ugly`: Remove all whitespace from JSON. - `@pretty`: Make the JSON more human readable. -- `@this`: Returns the current element. Can be used to retrieve the root element. +- `@this`: Returns the current element. It can be used to retrieve the root element. +- `@valid`: Ensure the json document is valid. +- `@flatten`: Flattens an array. +- `@join`: Joins multiple objects into a single object. #### Modifier arguments diff --git a/gjson.go b/gjson.go index bac29a9..647ed73 100644 --- a/gjson.go +++ b/gjson.go @@ -2595,6 +2595,15 @@ func execModifier(json, path string) (pathOut, res string, ok bool) { return pathOut, res, false } +// unwrap removes the '[]' or '{}' characters around json +func unwrap(json string) string { + json = trim(json) + if len(json) >= 2 && json[0] == '[' || json[0] == '{' { + json = json[1 : len(json)-1] + } + return json +} + // DisableModifiers will disable the modifier syntax var DisableModifiers = false @@ -2603,6 +2612,9 @@ var modifiers = map[string]func(json, arg string) string{ "ugly": modUgly, "reverse": modReverse, "this": modThis, + "flatten": modFlatten, + "join": modJoin, + "valid": modValid, } // AddModifier binds a custom modifier command to the GJSON syntax. @@ -2691,3 +2703,122 @@ func modReverse(json, arg string) string { } return json } + +// @flatten an array with child arrays. +// [1,[2],[3,4],[5,[6,7]]] -> [1,2,3,4,5,[6,7]] +// The {"deep":true} arg can be provide for deep flattening. +// [1,[2],[3,4],[5,[6,7]]] -> [1,2,3,4,5,6,7] +// The original json is returned when the json is not an array. +func modFlatten(json, arg string) string { + res := Parse(json) + if !res.IsArray() { + return json + } + var deep bool + if arg != "" { + Parse(arg).ForEach(func(key, value Result) bool { + if key.String() == "deep" { + deep = value.Bool() + } + return true + }) + } + var out []byte + out = append(out, '[') + var idx int + res.ForEach(func(_, value Result) bool { + if idx > 0 { + out = append(out, ',') + } + if value.IsArray() { + if deep { + out = append(out, unwrap(modFlatten(value.Raw, arg))...) + } else { + out = append(out, unwrap(value.Raw)...) + } + } else { + out = append(out, value.Raw...) + } + idx++ + return true + }) + out = append(out, ']') + return bytesString(out) +} + +// @join multiple objects into a single object. +// [{"first":"Tom"},{"last":"Smith"}] -> {"first","Tom","last":"Smith"} +// The arg can be "true" to specify that duplicate keys should be preserved. +// [{"first":"Tom","age":37},{"age":41}] -> {"first","Tom","age":37,"age":41} +// Without preserved keys: +// [{"first":"Tom","age":37},{"age":41}] -> {"first","Tom","age":41} +// The original json is returned when the json is not an object. +func modJoin(json, arg string) string { + res := Parse(json) + if !res.IsArray() { + return json + } + var preserve bool + if arg != "" { + Parse(arg).ForEach(func(key, value Result) bool { + if key.String() == "preserve" { + preserve = value.Bool() + } + return true + }) + } + var out []byte + out = append(out, '{') + if preserve { + // Preserve duplicate keys. + var idx int + res.ForEach(func(_, value Result) bool { + if !value.IsObject() { + return true + } + if idx > 0 { + out = append(out, ',') + } + out = append(out, unwrap(value.Raw)...) + idx++ + return true + }) + } else { + // Deduplicate keys and generate an object with stable ordering. + var keys []Result + kvals := make(map[string]Result) + res.ForEach(func(_, value Result) bool { + if !value.IsObject() { + return true + } + value.ForEach(func(key, value Result) bool { + k := key.String() + if _, ok := kvals[k]; !ok { + keys = append(keys, key) + } + kvals[k] = value + return true + }) + return true + }) + for i := 0; i < len(keys); i++ { + if i > 0 { + out = append(out, ',') + } + out = append(out, keys[i].Raw...) + out = append(out, ':') + out = append(out, kvals[keys[i].String()].Raw...) + } + } + out = append(out, '}') + return bytesString(out) +} + +// @valid ensures that the json is valid before moving on. An empty string is +// returned when the json is not valid, otherwise it returns the original json. +func modValid(json, arg string) string { + if !Valid(json) { + return "" + } + return json +} diff --git a/gjson_test.go b/gjson_test.go index 7a3255a..36a4273 100644 --- a/gjson_test.go +++ b/gjson_test.go @@ -2025,3 +2025,140 @@ func TestChainedModifierStringArgs(t *testing.T) { res := Get("[]", `@push:"2"|@push:"3"|@push:{"a":"b","c":["e","f"]}|@push:true|@push:10.23`) assert(t, res.String() == `["2","3",{"a":"b","c":["e","f"]},true,10.23]`) } + +func TestFlatten(t *testing.T) { + json := `[1,[2],[3,4],[5,[6,[7]]],{"hi":"there"},8,[9]]` + assert(t, Get(json, "@flatten").String() == `[1,2,3,4,5,[6,[7]],{"hi":"there"},8,9]`) + assert(t, Get(json, `@flatten:{"deep":true}`).String() == `[1,2,3,4,5,6,7,{"hi":"there"},8,9]`) + assert(t, Get(`{"9999":1234}`, "@flatten").String() == `{"9999":1234}`) +} + +func TestJoin(t *testing.T) { + assert(t, Get(`[{},{}]`, "@join").String() == `{}`) + assert(t, Get(`[{"a":1},{"b":2}]`, "@join").String() == `{"a":1,"b":2}`) + assert(t, Get(`[{"a":1,"b":1},{"b":2}]`, "@join").String() == `{"a":1,"b":2}`) + assert(t, Get(`[{"a":1,"b":1},{"b":2},5,{"c":3}]`, "@join").String() == `{"a":1,"b":2,"c":3}`) + assert(t, Get(`[{"a":1,"b":1},{"b":2},5,{"c":3}]`, `@join:{"preserve":true}`).String() == `{"a":1,"b":1,"b":2,"c":3}`) + assert(t, Get(`[{"a":1,"b":1},{"b":2},5,{"c":3}]`, `@join:{"preserve":true}.b`).String() == `1`) + assert(t, Get(`{"9999":1234}`, "@join").String() == `{"9999":1234}`) +} + +func TestValid(t *testing.T) { + assert(t, Get("[{}", "@valid").Exists() == false) + assert(t, Get("[{}]", "@valid").Exists() == true) +} + +// https://github.com/tidwall/gjson/issues/152 +func TestJoin152(t *testing.T) { + var json = `{ + "distance": 1374.0, + "validFrom": "2005-11-14", + "historical": { + "type": "Day", + "name": "last25Hours", + "summary": { + "units": { + "temperature": "C", + "wind": "m/s", + "snow": "cm", + "precipitation": "mm" + }, + "days": [ + { + "time": "2020-02-08", + "hours": [ + { + "temperature": { + "min": -2.0, + "max": -1.6, + "value": -1.6 + }, + "wind": {}, + "precipitation": {}, + "humidity": { + "value": 92.0 + }, + "snow": { + "depth": 49.0 + }, + "time": "2020-02-08T16:00:00+01:00" + }, + { + "temperature": { + "min": -1.7, + "max": -1.3, + "value": -1.3 + }, + "wind": {}, + "precipitation": {}, + "humidity": { + "value": 92.0 + }, + "snow": { + "depth": 49.0 + }, + "time": "2020-02-08T17:00:00+01:00" + }, + { + "temperature": { + "min": -1.3, + "max": -0.9, + "value": -1.2 + }, + "wind": {}, + "precipitation": {}, + "humidity": { + "value": 91.0 + }, + "snow": { + "depth": 49.0 + }, + "time": "2020-02-08T18:00:00+01:00" + } + ] + }, + { + "time": "2020-02-09", + "hours": [ + { + "temperature": { + "min": -1.7, + "max": -0.9, + "value": -1.5 + }, + "wind": {}, + "precipitation": {}, + "humidity": { + "value": 91.0 + }, + "snow": { + "depth": 49.0 + }, + "time": "2020-02-09T00:00:00+01:00" + }, + { + "temperature": { + "min": -1.5, + "max": 0.9, + "value": 0.2 + }, + "wind": {}, + "precipitation": {}, + "humidity": { + "value": 67.0 + }, + "snow": { + "depth": 49.0 + }, + "time": "2020-02-09T01:00:00+01:00" + } + ] + } + ] + } + } + }` + + res := Get(json, "historical.summary.days.#.hours|@flatten|#.humidity.value") + assert(t, res.Raw == `[92.0,92.0,91.0,91.0,67.0]`) +}