Skip to content

Commit 2cd10a2

Browse files
kenrickyapkenrickyap
committed
added integ-test and doc test
Signed-off-by: Kenrick Yap <[email protected]>
1 parent 018e462 commit 2cd10a2

File tree

5 files changed

+97
-18
lines changed

5 files changed

+97
-18
lines changed

‎core/src/main/java/org/opensearch/sql/utils/JsonUtils.java

+4-5
Original file line numberDiff line numberDiff line change
@@ -8,21 +8,16 @@
88
import com.fasterxml.jackson.databind.JsonNode;
99
import com.fasterxml.jackson.databind.ObjectMapper;
1010
import com.jayway.jsonpath.Configuration;
11-
import com.jayway.jsonpath.DocumentContext;
1211
import com.jayway.jsonpath.InvalidJsonException;
1312
import com.jayway.jsonpath.JsonPath;
1413

15-
import java.io.ObjectInputFilter;
1614
import java.util.LinkedHashMap;
1715
import java.util.LinkedList;
1816
import java.util.List;
1917
import java.util.Map;
20-
import java.util.stream.Collectors;
2118

2219
import com.jayway.jsonpath.Option;
2320
import com.jayway.jsonpath.PathNotFoundException;
24-
import com.jayway.jsonpath.spi.json.JacksonJsonProvider;
25-
import com.jayway.jsonpath.spi.mapper.JacksonMappingProvider;
2621
import lombok.experimental.UtilityClass;
2722
import org.opensearch.sql.data.model.ExprCollectionValue;
2823
import org.opensearch.sql.data.model.ExprDoubleValue;
@@ -66,6 +61,10 @@ public static ExprValue extractJson(ExprValue json, ExprValue path) {
6661
String jsonString = json.stringValue();
6762
String jsonPath = path.stringValue();
6863

64+
if (jsonString.equals("")) {
65+
return LITERAL_NULL;
66+
}
67+
6968
try {
7069
Configuration config = Configuration.builder()
7170
.options(Option.AS_PATH_LIST)

‎docs/user/ppl/functions/json.rst

+57-10
Original file line numberDiff line numberDiff line change
@@ -24,16 +24,17 @@ Example::
2424

2525
> source=json_test | eval is_valid = json_valid(json_string) | fields test_name, json_string, is_valid
2626
fetched rows / total rows = 6/6
27-
+---------------------+---------------------------------+----------+
28-
| test_name | json_string | is_valid |
29-
|---------------------|---------------------------------|----------|
30-
| json nested object | {"a":"1","b":{"c":"2","d":"3"}} | True |
31-
| json object | {"a":"1","b":"2"} | True |
32-
| json array | [1, 2, 3, 4] | True |
33-
| json scalar string | "abc" | True |
34-
| json empty string | | True |
35-
| json invalid object | {"invalid":"json", "string"} | False |
36-
+---------------------+---------------------------------+----------+
27+
+---------------------+-------------------------------------+----------+
28+
| test_name | json_string | is_valid |
29+
|---------------------|-------------------------------------|----------|
30+
| json nested object | {"a":"1","b":{"c":"2","d":"3"}} | True |
31+
| json nested list | {"a":"1","b":[{"c":"2"},{"c":"3"}]} | True |
32+
| json object | {"a":"1","b":"2"} | True |
33+
| json array | [1, 2, 3, 4] | True |
34+
| json scalar string | "abc" | True |
35+
| json empty string | | True |
36+
| json invalid object | {"invalid":"json", "string"} | False |
37+
+---------------------+-------------------------------------+----------+
3738

3839
JSON
3940
----------
@@ -59,3 +60,49 @@ Example::
5960
| json scalar string | "abc" | "abc" |
6061
| json empty string | | null |
6162
+---------------------+------------------------------+---------------+
63+
64+
JSON_EXTRACT
65+
____________
66+
67+
Description
68+
>>>>>>>>>>>
69+
70+
Usage: `json_extract(doc, path [, path]...)` Extracts a json value or scalar from a json document based on the path(s) specified.
71+
72+
Argument type: STRING, STRING
73+
74+
Return type: BOOLEAN/DOUBLE/INTEGER/NULL/STRUCT/ARRAY
75+
76+
- Returns a JSON array for multiple paths or if the path leads to an array.
77+
- Return null if path is not valid.
78+
79+
Example::
80+
81+
> source=json_test | where json_valid(json_string) | eval json_extract=json_extract(json_string, '$.b') | fields test_name, json_string, json_extract
82+
fetched rows / total rows = 6/6
83+
+---------------------+-------------------------------------+-------------------+
84+
| test_name | json_string | json_extract |
85+
|---------------------|-------------------------------------|-------------------|
86+
| json nested object | {"a":"1","b":{"c":"2","d":"3"}} | {c:"2",d:"3"} |
87+
| json nested list | {"a":"1","b":[{"c":"2"},{"c":"3"}]} | [{c:"2"},{c:"3"}] |
88+
| json object | {"a":"1","b":"2"} | 2 |
89+
| json array | [1, 2, 3, 4] | null |
90+
| json scalar string | "abc" | null |
91+
| json empty string | | null |
92+
+---------------------+-------------------------------------+-------------------+
93+
94+
> source=json_test | where test_name="json nested list" | eval json_extract=json_extract('{"a":[{"b":1},{"b":2}]}', '$.b[1].c')
95+
fetched rows / total rows = 1/1
96+
+---------------------+-------------------------------------+--------------+
97+
| test_name | json_string | json_extract |
98+
|---------------------|-------------------------------------|--------------|
99+
| json nested list | {"a":"1","b":[{"c":"2"},{"c":"3"}]} | 3 |
100+
+---------------------+-------------------------------------+--------------+
101+
102+
> source=json_test | where test_name="json nested list" | eval json_extract=json_extract('{"a":[{"b":1},{"b":2}]}', '$.b[*].c')
103+
fetched rows / total rows = 1/1
104+
+---------------------+-------------------------------------+--------------+
105+
| test_name | json_string | json_extract |
106+
|---------------------|-------------------------------------|--------------|
107+
| json nested list | {"a":"1","b":[{"c":"2"},{"c":"3"}]} | [2,3] |
108+
+---------------------+-------------------------------------+--------------+

‎doctest/test_data/json_test.json

+1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
{"test_name":"json nested object", "json_string":"{\"a\":\"1\",\"b\":{\"c\":\"2\",\"d\":\"3\"}}"}
2+
{"test_name":"json nested list", "json_string":"{\"a\":\"1\",\"b\":[{\"c\":\"2\"}, {\"c\":\"3\"}]}"}
23
{"test_name":"json object", "json_string":"{\"a\":\"1\",\"b\":\"2\"}"}
34
{"test_name":"json array", "json_string":"[1, 2, 3, 4]"}
45
{"test_name":"json scalar string", "json_string":"\"abc\""}

‎integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java

+33-3
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,9 @@ public void test_json_valid() throws IOException {
4040
rows("json object"),
4141
rows("json array"),
4242
rows("json scalar string"),
43-
rows("json empty string"));
43+
rows("json empty string"),
44+
rows("json nested list")
45+
);
4446
}
4547

4648
@Test
@@ -73,7 +75,10 @@ public void test_cast_json() throws IOException {
7375
rows("json object", new JSONObject(Map.of("a", "1", "b", "2"))),
7476
rows("json array", new JSONArray(List.of(1, 2, 3, 4))),
7577
rows("json scalar string", "abc"),
76-
rows("json empty string", null));
78+
rows("json empty string", null),
79+
rows("json nested list",
80+
new JSONObject(Map.of("a", "1", "b", List.of(Map.of("c", "2"), Map.of("c", "3")))))
81+
);
7782
}
7883

7984
@Test
@@ -96,6 +101,31 @@ public void test_json() throws IOException {
96101
rows("json object", new JSONObject(Map.of("a", "1", "b", "2"))),
97102
rows("json array", new JSONArray(List.of(1, 2, 3, 4))),
98103
rows("json scalar string", "abc"),
99-
rows("json empty string", null));
104+
rows("json empty string", null),
105+
rows("json nested list",
106+
new JSONObject(Map.of("a", "1", "b", List.of(Map.of("c", "2"), Map.of("c", "3")))))
107+
);
108+
}
109+
110+
@Test
111+
public void test_json_extract() throws IOException {
112+
JSONObject result;
113+
114+
result =
115+
executeQuery(
116+
String.format(
117+
"source=%s | where json_valid(json_string) | eval json_extract=json_extract(json_string, '$.b') | fields"
118+
+ " test_name, json_extract",
119+
TEST_INDEX_JSON_TEST));
120+
verifySchema(result, schema("test_name", null, "string"), schema("json_extract", null, "undefined"));
121+
verifyDataRows(
122+
result,
123+
rows("json nested object", new JSONObject(Map.of("c", "3"))),
124+
rows("json object", "2"),
125+
rows("json array", null),
126+
rows("json scalar string", null),
127+
rows("json empty string", null),
128+
rows("json nested list", new JSONArray(List.of(Map.of("c","2"), Map.of("c","3"))))
129+
);
100130
}
101131
}

‎integ-test/src/test/resources/json_test.json

+2
Original file line numberDiff line numberDiff line change
@@ -12,3 +12,5 @@
1212
{"test_name":"json invalid object", "json_string":"{\"invalid\":\"json\", \"string\"}"}
1313
{"index":{"_id":"6"}}
1414
{"test_name":"json null", "json_string":null}
15+
{"index":{"_id":"7"}}
16+
{"test_name":"json nested list", "json_string":"{\"a\":\"1\",\"b\":[{\"c\":\"2\"}, {\"c\":\"3\"}]}"}

0 commit comments

Comments
 (0)