diff --git a/core/src/main/java/org/opensearch/sql/expression/DSL.java b/core/src/main/java/org/opensearch/sql/expression/DSL.java index cde00fcc92..950fdcef87 100644 --- a/core/src/main/java/org/opensearch/sql/expression/DSL.java +++ b/core/src/main/java/org/opensearch/sql/expression/DSL.java @@ -691,6 +691,10 @@ public static FunctionExpression stringToJson(Expression value) { return compile(FunctionProperties.None, BuiltinFunctionName.JSON, value); } + public static FunctionExpression jsonObject(Expression... expressions) { + return compile(FunctionProperties.None, BuiltinFunctionName.JSON_OBJECT, expressions); + } + public static Aggregator avg(Expression... expressions) { return aggregate(BuiltinFunctionName.AVG, expressions); } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index 366321bed2..56a1791d5e 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -207,6 +207,7 @@ public enum BuiltinFunctionName { /** Json Functions. */ JSON_VALID(FunctionName.of("json_valid")), JSON(FunctionName.of("json")), + JSON_OBJECT(FunctionName.of("json_object")), /** GEOSPATIAL Functions. */ GEOIP(FunctionName.of("geoip")), diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index 75f134aa4e..885380578c 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -7,15 +7,33 @@ import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; import static org.opensearch.sql.data.type.ExprCoreType.STRING; +import static org.opensearch.sql.data.type.ExprCoreType.STRUCT; import static org.opensearch.sql.data.type.ExprCoreType.UNDEFINED; +import static org.opensearch.sql.expression.DSL.jsonObject; import static org.opensearch.sql.expression.function.FunctionDSL.define; import static org.opensearch.sql.expression.function.FunctionDSL.impl; import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandling; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; import lombok.experimental.UtilityClass; +import org.apache.commons.lang3.tuple.Pair; +import org.opensearch.sql.data.model.ExprTupleValue; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.type.ExprCoreType; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.exception.SemanticCheckException; +import org.opensearch.sql.expression.Expression; +import org.opensearch.sql.expression.FunctionExpression; +import org.opensearch.sql.expression.env.Environment; import org.opensearch.sql.expression.function.BuiltinFunctionName; import org.opensearch.sql.expression.function.BuiltinFunctionRepository; import org.opensearch.sql.expression.function.DefaultFunctionResolver; +import org.opensearch.sql.expression.function.FunctionBuilder; +import org.opensearch.sql.expression.function.FunctionName; +import org.opensearch.sql.expression.function.FunctionResolver; +import org.opensearch.sql.expression.function.FunctionSignature; import org.opensearch.sql.utils.JsonUtils; @UtilityClass @@ -23,6 +41,7 @@ public class JsonFunctions { public void register(BuiltinFunctionRepository repository) { repository.register(jsonValid()); repository.register(jsonFunction()); + repository.register(jsonObject()); } private DefaultFunctionResolver jsonValid() { @@ -35,4 +54,61 @@ private DefaultFunctionResolver jsonFunction() { BuiltinFunctionName.JSON.getName(), impl(nullMissingHandling(JsonUtils::castJson), UNDEFINED, STRING)); } + + /** Creates a JSON Object/tuple expr from a given list of kv pairs. */ + private static FunctionResolver jsonObject() { + return new FunctionResolver() { + @Override + public FunctionName getFunctionName() { + return BuiltinFunctionName.JSON_OBJECT.getName(); + } + + @Override + public Pair resolve( + FunctionSignature unresolvedSignature) { + List paramList = unresolvedSignature.getParamTypeList(); + // check that we got an even number of arguments + if (paramList.size() % 2 != 0) { + throw new SemanticCheckException( + String.format( + "Expected an even number of arguments but instead got %d arguments", + paramList.size())); + } + + // check that each "key" argument (of key-value pair) is a string + for (int i = 0; i < paramList.size(); i = i + 2) { + ExprType paramType = paramList.get(i); + if (!ExprCoreType.STRING.equals(paramType)) { + throw new SemanticCheckException( + String.format( + "Expected type %s instead of %s for parameter #%d", + ExprCoreType.STRING, paramType.typeName(), i + 1)); + } + } + + // return the unresolved signature and function builder + return Pair.of( + unresolvedSignature, + (functionProperties, arguments) -> + new FunctionExpression(getFunctionName(), arguments) { + @Override + public ExprValue valueOf(Environment valueEnv) { + LinkedHashMap tupleValues = new LinkedHashMap<>(); + Iterator iter = getArguments().iterator(); + while (iter.hasNext()) { + tupleValues.put( + iter.next().valueOf(valueEnv).stringValue(), + iter.next().valueOf(valueEnv)); + } + return ExprTupleValue.fromExprValueMap(tupleValues); + } + + @Override + public ExprType type() { + return STRUCT; + } + }); + } + }; + } } diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index bba8475c11..c58e6f012b 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -194,27 +194,91 @@ void json_returnsScalar() { @Test void json_returnsSemanticCheckException() { - List expressions = - List.of( - DSL.literal("invalid"), // invalid type - DSL.literal("{{[}}"), // missing bracket - DSL.literal("[}"), // missing bracket - DSL.literal("}"), // missing bracket - DSL.literal("\"missing quote"), // missing quote - DSL.literal("abc"), // not a type - DSL.literal("97ab"), // not a type - DSL.literal("{1, 2, 3, 4}"), // invalid object - DSL.literal("{123: 1, true: 2, null: 3}"), // invalid object - DSL.literal("{\"invalid\":\"json\", \"string\"}"), // invalid object - DSL.literal("[\"a\": 1, \"b\": 2]") // invalid array - ); + // invalid type + assertThrows( + SemanticCheckException.class, () -> DSL.castJson(DSL.literal("invalid")).valueOf()); - expressions.stream() - .forEach( - expr -> - assertThrows( - SemanticCheckException.class, - () -> DSL.castJson(expr).valueOf(), - "Expected to throw SemanticCheckException when calling castJson with " + expr)); + // missing bracket + assertThrows(SemanticCheckException.class, () -> DSL.castJson(DSL.literal("{{[}}")).valueOf()); + + // missing quote + assertThrows( + SemanticCheckException.class, () -> DSL.castJson(DSL.literal("\"missing quote")).valueOf()); + } + + @Test + public void json_object_returns_tuple() { + FunctionExpression exp; + + // Setup + LinkedHashMap objectMap = new LinkedHashMap<>(); + objectMap.put("foo", new ExprStringValue("foo")); + objectMap.put("fuzz", ExprBooleanValue.of(true)); + objectMap.put("bar", new ExprLongValue(1234)); + objectMap.put("bar2", new ExprDoubleValue(12.34)); + objectMap.put("baz", ExprNullValue.of()); + objectMap.put( + "obj", ExprTupleValue.fromExprValueMap(Map.of("internal", new ExprStringValue("value")))); + // TODO: requires json_array() + // objectMap.put( + // "arr", + // new ExprCollectionValue( + // List.of(new ExprStringValue("string"), ExprBooleanValue.of(true), + // ExprNullValue.of()))); + ExprValue expectedTupleExpr = ExprTupleValue.fromExprValueMap(objectMap); + + // exercise + exp = + DSL.jsonObject( + DSL.literal("foo"), DSL.literal("foo"), + DSL.literal("fuzz"), DSL.literal(true), + DSL.literal("bar"), DSL.literal(1234), + DSL.literal("bar2"), DSL.literal(12.34), + DSL.literal("baz"), DSL.literal(LITERAL_NULL), + DSL.literal("obj"), DSL.jsonObject(DSL.literal("internal"), DSL.literal("value"))); + + // Verify + var value = exp.valueOf(); + assertTrue(value instanceof ExprTupleValue); + assertEquals(expectedTupleExpr, value); + } + + @Test + public void json_object_returns_empty_tuple() { + FunctionExpression exp; + + // Setup + LinkedHashMap objectMap = new LinkedHashMap<>(); + ExprValue expectedTupleExpr = ExprTupleValue.fromExprValueMap(objectMap); + + // exercise + exp = DSL.jsonObject(); + + // Verify + var value = exp.valueOf(); + assertTrue(value instanceof ExprTupleValue); + assertEquals(expectedTupleExpr, value); + } + + @Test + public void json_object_throws_SemanticCheckException() { + // wrong number of arguments + assertThrows( + SemanticCheckException.class, () -> DSL.jsonObject(DSL.literal("only one")).valueOf()); + assertThrows( + SemanticCheckException.class, + () -> + DSL.jsonObject(DSL.literal("one"), DSL.literal("two"), DSL.literal("three")).valueOf()); + + // key argument is not a string + assertThrows( + SemanticCheckException.class, + () -> DSL.jsonObject(DSL.literal(1234), DSL.literal("two")).valueOf()); + assertThrows( + SemanticCheckException.class, + () -> + DSL.jsonObject( + DSL.literal("one"), DSL.literal(true), DSL.literal(true), DSL.literal("four")) + .valueOf()); } } diff --git a/docs/user/ppl/functions/json.rst b/docs/user/ppl/functions/json.rst index 77d9d00f45..9cced28db3 100644 --- a/docs/user/ppl/functions/json.rst +++ b/docs/user/ppl/functions/json.rst @@ -60,3 +60,43 @@ Example:: | json scalar string | "abc" | "abc" | | json empty string | | null | +---------------------+---------------------------------+-------------------------+ + +JSON_OBJECT +----------- + +Description +>>>>>>>>>>> + +Usage: `json_object(, [, , ]...)` returns a JSON object from key-value pairs. + +Argument type: +- A \ must be STRING. +- A \ can be a scalar, another json object, or json array type. Note: scalar fields will be treated as single-value. Use `json_array` to construct an array value from a multi-value. + +Return type: STRUCT + +Example: + + os> source=people | eval result = json_object('key', 123.45) | fields result + fetched rows / total rows = 1/1 + +-----------------+ + | result | + |-----------------| + | {'key': 123.45} | + +-----------------+ + + os> source=people | eval result = json_object('outer', json_object('inner', 123.45)) | fields result + fetched rows / total rows = 1/1 + +------------------------------+ + | result | + |------------------------------| + | {'outer': {'inner': 123.45}} | + +------------------------------+ + + source=people | eval result = json_object('array_doc', json_array(123.45, "string", true, null)) | fields result + fetched rows / total rows = 1/1 + +------------------------------------------------+ + | result | + +------------------------------------------------+ + | {"array_doc":[123.45, "string", true, null]} | + +------------------------------------------------+ diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java index b6a2d5e4aa..87ff418dcb 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java @@ -183,4 +183,38 @@ public void test_cast_json_scalar_to_type() throws IOException { verifyDataRows( result, rows("json scalar boolean true", true), rows("json scalar boolean false", false)); } + + @Test + public void test_json_object() throws IOException { + JSONObject result; + + result = + executeQuery( + String.format( + "source=%s | where json_valid(json_string) | " + + "eval obj=json_object('key', json(json_string)) | " + + "fields test_name, obj", + TEST_INDEX_JSON_TEST)); + verifySchema(result, schema("test_name", null, "string"), schema("obj", null, "struct")); + verifyDataRows( + result, + rows( + "json nested object", + new JSONObject( + Map.of( + "key", + Map.of("a", "1", "b", Map.of("c", "3"), "d", List.of(Boolean.FALSE, 3))))), + rows("json object", new JSONObject(Map.of("key", Map.of("a", "1", "b", "2")))), + rows( + "json nested array", + new JSONObject(Map.of("key", List.of(1, 2, 3, Map.of("true", true, "number", 123))))), + rows("json array", new JSONObject(Map.of("key", List.of(1, 2, 3, 4)))), + rows("json scalar string", Map.of("key", "abc")), + rows("json scalar int", Map.of("key", 1234)), + rows("json scalar float", Map.of("key", 12.34)), + rows("json scalar double", Map.of("key", 2.99792458e8)), + rows("json scalar boolean true", Map.of("key", true)), + rows("json scalar boolean false", Map.of("key", false)), + rows("json empty string", Map.of())); + } } diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 0307fb4ca1..9629500ba5 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -334,6 +334,7 @@ CIDRMATCH: 'CIDRMATCH'; // JSON FUNCTIONS JSON_VALID: 'JSON_VALID'; JSON: 'JSON'; +JSON_OBJECT: 'JSON_OBJECT'; // FLOWCONTROL FUNCTIONS IFNULL: 'IFNULL'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 451edeb29b..b840379f71 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -310,6 +310,7 @@ valueExpression | extractFunction # extractFunctionCall | getFormatFunction # getFormatFunctionCall | timestampFunction # timestampFunctionCall + | jsonObjectFunction # jsonObjectFunctionCall | LT_PRTHS valueExpression RT_PRTHS # parentheticValueExpr ; @@ -324,6 +325,10 @@ positionFunction : positionFunctionName LT_PRTHS functionArg IN functionArg RT_PRTHS ; +jsonObjectFunction + : jsonObjectFunctionName LT_PRTHS (functionArg COMMA functionArg (COMMA functionArg COMMA functionArg)*)? RT_PRTHS + ; + booleanExpression : booleanFunctionCall ; @@ -420,6 +425,7 @@ evalFunctionName | flowControlFunctionName | systemFunctionName | positionFunctionName + | jsonObjectFunctionName | jsonFunctionName | geoipFunctionName ; @@ -707,6 +713,10 @@ positionFunctionName : POSITION ; +jsonObjectFunctionName + : JSON_OBJECT + ; + jsonFunctionName : JSON ;