From f82554b7e548797ce9a270d0f3f4261fbec684b2 Mon Sep 17 00:00:00 2001 From: Kenrick Yap <14yapkc1@gmail.com> Date: Fri, 3 Jan 2025 10:43:05 -0800 Subject: [PATCH 01/29] added implementation Signed-off-by: Kenrick Yap <14yapkc1@gmail.com> --- .../opensearch/sql/expression/json/JsonFunctions.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index 75f134aa4e..adcd5d8143 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -5,6 +5,15 @@ package org.opensearch.sql.expression.json; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.experimental.UtilityClass; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.model.ExprValueUtils; +import org.opensearch.sql.exception.SemanticCheckException; +import org.opensearch.sql.expression.function.BuiltinFunctionName; +import org.opensearch.sql.expression.function.BuiltinFunctionRepository; +import org.opensearch.sql.expression.function.DefaultFunctionResolver; + import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.data.type.ExprCoreType.UNDEFINED; From 08e16ff35c25904b02686346b7fd00f7df5dba62 Mon Sep 17 00:00:00 2001 From: Kenrick Yap <14yapkc1@gmail.com> Date: Mon, 6 Jan 2025 15:14:25 -0800 Subject: [PATCH 02/29] added doctest, integ-tests, and unit tests Signed-off-by: Kenrick Yap <14yapkc1@gmail.com> --- .../opensearch/sql/ppl/JsonFunctionIT.java | 65 +++++++++++++++++++ .../json_test_index_mappping.json | 12 ++++ 2 files changed, 77 insertions(+) create mode 100644 integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java create mode 100644 integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java new file mode 100644 index 0000000000..62e7868b41 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java @@ -0,0 +1,65 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ppl; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import org.json.JSONObject; + +import javax.json.Json; + +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_JSON_TEST; +import static org.opensearch.sql.util.MatcherUtils.rows; +import static org.opensearch.sql.util.MatcherUtils.schema; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; +import static org.opensearch.sql.util.MatcherUtils.verifySchema; + +public class JsonFunctionIT extends PPLIntegTestCase { + @Override + public void init() throws IOException { + loadIndex(Index.JSON_TEST); + } + + @Test + public void test_json_valid() throws IOException { + JSONObject result; + + result = + executeQuery( + String.format( + "source=%s | where json_valid(json_string) | fields test_name", + TEST_INDEX_JSON_TEST + ) + ); + verifySchema(result, schema("test_name", null, "string")); + verifyDataRows( + result, + rows("json object"), + rows("json array"), + rows("json scalar string"), + rows("json empty string") + ); + } + + @Test + public void test_not_json_valid() throws IOException { + JSONObject result; + + result = + executeQuery( + String.format( + "source=%s | where not json_valid(json_string) | fields test_name", + TEST_INDEX_JSON_TEST + ) + ); + verifySchema(result, schema("test_name", null, "string")); + verifyDataRows( + result, + rows("json invalid object") + ); + } +} diff --git a/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json b/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json new file mode 100644 index 0000000000..b825254b11 --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json @@ -0,0 +1,12 @@ +{ + "mappings": { + "properties": { + "test_name": { + "type": "text" + }, + "json_string": { + "type": "text" + } + } + } +} From ebb5cc31038e4e380505157f58b0055cc2ece9d4 Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Tue, 7 Jan 2025 09:20:59 -0800 Subject: [PATCH 03/29] addressed pr comments Signed-off-by: Kenrick Yap --- .../resources/indexDefinitions/json_test_index_mappping.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json b/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json index b825254b11..fb97836d5e 100644 --- a/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json +++ b/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json @@ -2,7 +2,7 @@ "mappings": { "properties": { "test_name": { - "type": "text" + "type": "keyword" }, "json_string": { "type": "text" From b094decdea41ac2ce93deadb4a92559a0cc1bf29 Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Tue, 7 Jan 2025 13:40:28 -0800 Subject: [PATCH 04/29] addressed PR comments Signed-off-by: Kenrick Yap --- .../sql/expression/datetime/DateTimeFunctionTest.java | 2 ++ .../org/opensearch/sql/expression/datetime/ExtractTest.java | 3 +++ .../org/opensearch/sql/expression/datetime/YearweekTest.java | 3 +++ 3 files changed, 8 insertions(+) diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java index ad15dadfb7..e8a4fc81fb 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java @@ -29,6 +29,7 @@ import java.util.List; import java.util.stream.Stream; import lombok.AllArgsConstructor; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -1231,6 +1232,7 @@ public void testWeekFormats( } @Test + @Disabled("Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") public void testWeekOfYearWithTimeType() { LocalDate today = LocalDate.now(functionProperties.getQueryStartClock()); diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java index 77b65cbed2..53ad63d366 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java @@ -15,6 +15,8 @@ import java.time.ZoneId; import java.time.temporal.IsoFields; import java.util.stream.Stream; + +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -102,6 +104,7 @@ private void datePartWithTimeArgQuery( } @Test + @Disabled("Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") public void testExtractDatePartWithTimeType() { LocalDate now = LocalDate.now(functionProperties.getQueryStartClock()); diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java index d944f7c85c..4de5d3a341 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java @@ -16,6 +16,8 @@ import java.time.LocalDate; import java.time.temporal.ChronoUnit; import java.util.stream.Stream; + +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -100,6 +102,7 @@ public void testYearweekWithoutMode() { } @Test + @Disabled("Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") public void testYearweekWithTimeType() { int expected = getYearWeekBeforeSunday(LocalDate.now(functionProperties.getQueryStartClock())); From 373e53db0f403c48dac385618317ca2d702cc1ad Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Tue, 7 Jan 2025 13:40:47 -0800 Subject: [PATCH 05/29] removed unused dependencies Signed-off-by: Kenrick Yap --- .../java/org/opensearch/sql/expression/json/JsonFunctions.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index adcd5d8143..64f84c44f7 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -7,9 +7,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import lombok.experimental.UtilityClass; -import org.opensearch.sql.data.model.ExprValue; -import org.opensearch.sql.data.model.ExprValueUtils; -import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.function.BuiltinFunctionName; import org.opensearch.sql.expression.function.BuiltinFunctionRepository; import org.opensearch.sql.expression.function.DefaultFunctionResolver; From 18a3cb3d305cadc23cfaa7b3f51a048717dde8cc Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Tue, 7 Jan 2025 14:28:45 -0800 Subject: [PATCH 06/29] linting Signed-off-by: Kenrick Yap --- .../sql/expression/json/JsonFunctions.java | 3 +- .../datetime/DateTimeFunctionTest.java | 3 +- .../sql/expression/datetime/ExtractTest.java | 4 +- .../sql/expression/datetime/YearweekTest.java | 4 +- .../opensearch/sql/ppl/JsonFunctionIT.java | 89 ++++++++----------- 5 files changed, 47 insertions(+), 56 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index 64f84c44f7..2227d8fe09 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -33,7 +33,8 @@ public void register(BuiltinFunctionRepository repository) { private DefaultFunctionResolver jsonValid() { return define( - BuiltinFunctionName.JSON_VALID.getName(), impl(JsonUtils::isValidJson, BOOLEAN, STRING)); + BuiltinFunctionName.JSON_VALID.getName(), + impl(nullMissingHandling(JsonUtils::isValidJson), BOOLEAN, STRING)); } private DefaultFunctionResolver jsonFunction() { diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java index e8a4fc81fb..eb5074f4f7 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java @@ -1232,7 +1232,8 @@ public void testWeekFormats( } @Test - @Disabled("Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") + @Disabled( + "Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") public void testWeekOfYearWithTimeType() { LocalDate today = LocalDate.now(functionProperties.getQueryStartClock()); diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java index 53ad63d366..10e60ee7e4 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java @@ -15,7 +15,6 @@ import java.time.ZoneId; import java.time.temporal.IsoFields; import java.util.stream.Stream; - import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -104,7 +103,8 @@ private void datePartWithTimeArgQuery( } @Test - @Disabled("Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") + @Disabled( + "Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") public void testExtractDatePartWithTimeType() { LocalDate now = LocalDate.now(functionProperties.getQueryStartClock()); diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java index 4de5d3a341..daae8b1ff5 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java @@ -16,7 +16,6 @@ import java.time.LocalDate; import java.time.temporal.ChronoUnit; import java.util.stream.Stream; - import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -102,7 +101,8 @@ public void testYearweekWithoutMode() { } @Test - @Disabled("Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") + @Disabled( + "Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") public void testYearweekWithTimeType() { int expected = getYearWeekBeforeSunday(LocalDate.now(functionProperties.getQueryStartClock())); diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java index 62e7868b41..f02750147d 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java @@ -5,61 +5,50 @@ package org.opensearch.sql.ppl; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import org.json.JSONObject; - -import javax.json.Json; - import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_JSON_TEST; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; import static org.opensearch.sql.util.MatcherUtils.verifySchema; -public class JsonFunctionIT extends PPLIntegTestCase { - @Override - public void init() throws IOException { - loadIndex(Index.JSON_TEST); - } - - @Test - public void test_json_valid() throws IOException { - JSONObject result; - - result = - executeQuery( - String.format( - "source=%s | where json_valid(json_string) | fields test_name", - TEST_INDEX_JSON_TEST - ) - ); - verifySchema(result, schema("test_name", null, "string")); - verifyDataRows( - result, - rows("json object"), - rows("json array"), - rows("json scalar string"), - rows("json empty string") - ); - } - - @Test - public void test_not_json_valid() throws IOException { - JSONObject result; +import java.io.IOException; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; - result = - executeQuery( - String.format( - "source=%s | where not json_valid(json_string) | fields test_name", - TEST_INDEX_JSON_TEST - ) - ); - verifySchema(result, schema("test_name", null, "string")); - verifyDataRows( - result, - rows("json invalid object") - ); - } +public class JsonFunctionIT extends PPLIntegTestCase { + @Override + public void init() throws IOException { + loadIndex(Index.JSON_TEST); + } + + @Test + public void test_json_valid() throws IOException { + JSONObject result; + + result = + executeQuery( + String.format( + "source=%s | where json_valid(json_string) | fields test_name", + TEST_INDEX_JSON_TEST)); + verifySchema(result, schema("test_name", null, "string")); + verifyDataRows( + result, + rows("json object"), + rows("json array"), + rows("json scalar string"), + rows("json empty string")); + } + + @Test + public void test_not_json_valid() throws IOException { + JSONObject result; + + result = + executeQuery( + String.format( + "source=%s | where not json_valid(json_string) | fields test_name", + TEST_INDEX_JSON_TEST)); + verifySchema(result, schema("test_name", null, "string")); + verifyDataRows(result, rows("json invalid object")); + } } From 372fd33408ff8ed3fab1da60c9ab5119c27babfd Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Wed, 8 Jan 2025 14:16:31 -0800 Subject: [PATCH 07/29] addressed pr comment and rolling back disabled test case Signed-off-by: Kenrick Yap --- .../sql/expression/datetime/DateTimeFunctionTest.java | 4 ++-- .../org/opensearch/sql/expression/datetime/ExtractTest.java | 2 -- .../org/opensearch/sql/expression/datetime/YearweekTest.java | 4 ++-- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java index eb5074f4f7..ab391c6834 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java @@ -1231,9 +1231,9 @@ public void testWeekFormats( expectedInteger); } + // subtracting 1 as a temporary fix for year 2024. + // Issue: https://github.com/opensearch-project/sql/issues/2477 @Test - @Disabled( - "Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") public void testWeekOfYearWithTimeType() { LocalDate today = LocalDate.now(functionProperties.getQueryStartClock()); diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java index 10e60ee7e4..4f7c3d71e3 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java @@ -103,8 +103,6 @@ private void datePartWithTimeArgQuery( } @Test - @Disabled( - "Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") public void testExtractDatePartWithTimeType() { LocalDate now = LocalDate.now(functionProperties.getQueryStartClock()); diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java index daae8b1ff5..266994c046 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java @@ -100,9 +100,9 @@ public void testYearweekWithoutMode() { assertEquals(eval(expression), eval(expressionWithoutMode)); } + // subtracting 1 as a temporary fix for year 2024. + // Issue: https://github.com/opensearch-project/sql/issues/2477 @Test - @Disabled( - "Test is disabled because of issue https://github.com/opensearch-project/sql/issues/2477") public void testYearweekWithTimeType() { int expected = getYearWeekBeforeSunday(LocalDate.now(functionProperties.getQueryStartClock())); From be33ff4d046da9848c857ebcc225ff0262c1b9f1 Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Thu, 9 Jan 2025 10:58:21 -0800 Subject: [PATCH 08/29] removed disabled import Signed-off-by: Kenrick Yap --- .../opensearch/sql/expression/datetime/DateTimeFunctionTest.java | 1 - .../java/org/opensearch/sql/expression/datetime/ExtractTest.java | 1 - .../org/opensearch/sql/expression/datetime/YearweekTest.java | 1 - 3 files changed, 3 deletions(-) diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java index ab391c6834..115898e349 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java @@ -29,7 +29,6 @@ import java.util.List; import java.util.stream.Stream; import lombok.AllArgsConstructor; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java index 4f7c3d71e3..77b65cbed2 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/ExtractTest.java @@ -15,7 +15,6 @@ import java.time.ZoneId; import java.time.temporal.IsoFields; import java.util.stream.Stream; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java index 266994c046..ee4df23be4 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java @@ -16,7 +16,6 @@ import java.time.LocalDate; import java.time.temporal.ChronoUnit; import java.util.stream.Stream; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; From b3a5eae1e8ae4fed10ee7bed527cb2a14234abc1 Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Thu, 9 Jan 2025 13:05:12 -0800 Subject: [PATCH 09/29] nit Signed-off-by: Kenrick Yap --- .../indexDefinitions/json_test_index_mappping.json | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json diff --git a/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json b/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json deleted file mode 100644 index fb97836d5e..0000000000 --- a/integ-test/src/test/resources/indexDefinitions/json_test_index_mappping.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "mappings": { - "properties": { - "test_name": { - "type": "keyword" - }, - "json_string": { - "type": "text" - } - } - } -} From 6397af41e2e66b000ae592c71f259803b97bd04e Mon Sep 17 00:00:00 2001 From: kenrickyap <121634635+kenrickyap@users.noreply.github.com> Date: Thu, 9 Jan 2025 13:02:10 -0800 Subject: [PATCH 10/29] Update integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java Co-authored-by: Andrew Carbonetto Signed-off-by: kenrickyap <121634635+kenrickyap@users.noreply.github.com> --- .../src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java index f02750147d..501ef9448e 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java @@ -15,7 +15,7 @@ import org.json.JSONObject; import org.junit.jupiter.api.Test; -public class JsonFunctionIT extends PPLIntegTestCase { +public class JsonFunctionsIT extends PPLIntegTestCase { @Override public void init() throws IOException { loadIndex(Index.JSON_TEST); From 6d9c50acf15f7b0931b2baea2a50039708855352 Mon Sep 17 00:00:00 2001 From: Kenrick Yap Date: Thu, 9 Jan 2025 13:16:09 -0800 Subject: [PATCH 11/29] fixed integ test Signed-off-by: Kenrick Yap --- .../opensearch/sql/ppl/JsonFunctionIT.java | 54 ------------------- 1 file changed, 54 deletions(-) delete mode 100644 integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java deleted file mode 100644 index 501ef9448e..0000000000 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionIT.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.ppl; - -import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_JSON_TEST; -import static org.opensearch.sql.util.MatcherUtils.rows; -import static org.opensearch.sql.util.MatcherUtils.schema; -import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; -import static org.opensearch.sql.util.MatcherUtils.verifySchema; - -import java.io.IOException; -import org.json.JSONObject; -import org.junit.jupiter.api.Test; - -public class JsonFunctionsIT extends PPLIntegTestCase { - @Override - public void init() throws IOException { - loadIndex(Index.JSON_TEST); - } - - @Test - public void test_json_valid() throws IOException { - JSONObject result; - - result = - executeQuery( - String.format( - "source=%s | where json_valid(json_string) | fields test_name", - TEST_INDEX_JSON_TEST)); - verifySchema(result, schema("test_name", null, "string")); - verifyDataRows( - result, - rows("json object"), - rows("json array"), - rows("json scalar string"), - rows("json empty string")); - } - - @Test - public void test_not_json_valid() throws IOException { - JSONObject result; - - result = - executeQuery( - String.format( - "source=%s | where not json_valid(json_string) | fields test_name", - TEST_INDEX_JSON_TEST)); - verifySchema(result, schema("test_name", null, "string")); - verifyDataRows(result, rows("json invalid object")); - } -} From 5b1f6de41bd10d00d96b2d88527360123fdce22b Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Wed, 15 Jan 2025 09:12:53 -0800 Subject: [PATCH 12/29] json_valid: null and missing should return false Signed-off-by: Andrew Carbonetto --- .../java/org/opensearch/sql/expression/json/JsonFunctions.java | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index 2227d8fe09..90f92e35cf 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -16,7 +16,6 @@ import static org.opensearch.sql.data.type.ExprCoreType.UNDEFINED; import static org.opensearch.sql.expression.function.FunctionDSL.define; import static org.opensearch.sql.expression.function.FunctionDSL.impl; -import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandling; import lombok.experimental.UtilityClass; import org.opensearch.sql.expression.function.BuiltinFunctionName; From 182a266d8083aa16006933078f9ee8f60aed631a Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Wed, 8 Jan 2025 10:49:03 -0800 Subject: [PATCH 13/29] PPL: Add json and cast to json functions Signed-off-by: Andrew Carbonetto --- .../org/opensearch/sql/expression/DSL.java | 4 + .../function/BuiltinFunctionName.java | 5 +- .../operator/convert/TypeCastOperators.java | 1 + .../org/opensearch/sql/utils/JsonUtils.java | 28 ++++++ .../expression/json/JsonFunctionsTest.java | 91 +++++++++++++++++++ ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 1 + ppl/src/main/antlr/OpenSearchPPLParser.g4 | 2 + 7 files changed, 131 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/DSL.java b/core/src/main/java/org/opensearch/sql/expression/DSL.java index cde00fcc92..9246a31bb7 100644 --- a/core/src/main/java/org/opensearch/sql/expression/DSL.java +++ b/core/src/main/java/org/opensearch/sql/expression/DSL.java @@ -985,6 +985,10 @@ public static FunctionExpression geoip(Expression... args) { return compile(FunctionProperties.None, BuiltinFunctionName.GEOIP, args); } + public static FunctionExpression json_function(Expression value) { + return compile(FunctionProperties.None, BuiltinFunctionName.JSON, value); + } + @SuppressWarnings("unchecked") private static T compile( FunctionProperties functionProperties, BuiltinFunctionName bfn, Expression... args) { diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index 366321bed2..ed54d30214 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -264,7 +264,10 @@ public enum BuiltinFunctionName { MULTIMATCH(FunctionName.of("multimatch")), MULTIMATCHQUERY(FunctionName.of("multimatchquery")), WILDCARDQUERY(FunctionName.of("wildcardquery")), - WILDCARD_QUERY(FunctionName.of("wildcard_query")); + WILDCARD_QUERY(FunctionName.of("wildcard_query")), + + /* Json Functions. */ + JSON(FunctionName.of("json")); private final FunctionName name; diff --git a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java index c1391ac9ab..aaae000cda 100644 --- a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java @@ -22,6 +22,7 @@ import static org.opensearch.sql.expression.function.FunctionDSL.implWithProperties; import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandling; import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandlingWithProperties; +import static org.opensearch.sql.utils.JsonUtils.castJson; import java.util.Arrays; import java.util.stream.Collectors; diff --git a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java index f38fe59789..3c785f8b96 100644 --- a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java @@ -108,5 +108,33 @@ private static ExprValue processJsonNode(JsonNode jsonNode) { // in all other cases, return null return LITERAL_NULL; } + if (jsonNode.isIntegralNumber()) { + return new ExprIntegerValue(jsonNode.asLong()); + } + if (jsonNode.isBoolean()) { + return jsonNode.asBoolean() ? LITERAL_TRUE : LITERAL_FALSE; + } + if (jsonNode.isTextual()) { + return new ExprStringValue(jsonNode.asText()); + } + if (jsonNode.isArray()) { + List elements = new LinkedList<>(); + for (var iter = jsonNode.iterator(); iter.hasNext(); ) { + jsonNode = iter.next(); + elements.add(processJsonNode(jsonNode)); + } + return new ExprCollectionValue(elements); + } + if (jsonNode.isObject()) { + Map values = new LinkedHashMap<>(); + for (var iter = jsonNode.fields(); iter.hasNext(); ) { + Map.Entry entry = iter.next(); + values.put(entry.getKey(), processJsonNode(entry.getValue())); + } + return ExprTupleValue.fromExprValueMap(values); + } + + // in all other cases, return null + return LITERAL_NULL; } } diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index bba8475c11..b138315eea 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -11,6 +11,8 @@ import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_FALSE; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_MISSING; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_TRUE; import java.util.LinkedHashMap; @@ -217,4 +219,93 @@ void json_returnsSemanticCheckException() { () -> DSL.castJson(expr).valueOf(), "Expected to throw SemanticCheckException when calling castJson with " + expr)); } + + @Test + void json_returnsJsonObject() { + FunctionExpression exp; + + // Setup + final String objectJson = + "{\"foo\": \"foo\", \"fuzz\": true, \"bar\": 1234, \"bar2\": 12.34, \"baz\": null, " + + "\"obj\": {\"internal\": \"value\"}, \"arr\": [\"string\", true, null]}"; + + LinkedHashMap objectMap = new LinkedHashMap<>(); + objectMap.put("foo", new ExprStringValue("foo")); + objectMap.put("fuzz", ExprBooleanValue.of(true)); + objectMap.put("bar", new ExprLongValue(1234)); + objectMap.put("bar2", new ExprDoubleValue(12.34)); + objectMap.put("baz", ExprNullValue.of()); + objectMap.put( + "obj", ExprTupleValue.fromExprValueMap(Map.of("internal", new ExprStringValue("value")))); + objectMap.put( + "arr", + new ExprCollectionValue( + List.of(new ExprStringValue("string"), ExprBooleanValue.of(true), ExprNullValue.of()))); + ExprValue expectedTupleExpr = ExprTupleValue.fromExprValueMap(objectMap); + + // exercise + exp = DSL.json_function(DSL.literal(objectJson)); + + // Verify + var value = exp.valueOf(); + assertTrue(value instanceof ExprTupleValue); + assertEquals(expectedTupleExpr, value); + } + + @Test + void json_returnsJsonArray() { + FunctionExpression exp; + + // Setup + final String arrayJson = "[\"foo\", \"fuzz\", true, \"bar\", 1234, 12.34, null]"; + ExprValue expectedArrayExpr = + new ExprCollectionValue( + List.of( + new ExprStringValue("foo"), + new ExprStringValue("fuzz"), + LITERAL_TRUE, + new ExprStringValue("bar"), + new ExprIntegerValue(1234), + new ExprDoubleValue(12.34), + LITERAL_NULL)); + + // exercise + exp = DSL.json_function(DSL.literal(arrayJson)); + + // Verify + var value = exp.valueOf(); + assertTrue(value instanceof ExprCollectionValue); + assertEquals(expectedArrayExpr, value); + } + + @Test + void json_returnsScalar() { + assertEquals( + new ExprStringValue("foobar"), DSL.json_function(DSL.literal("\"foobar\"")).valueOf()); + + assertEquals(new ExprIntegerValue(1234), DSL.json_function(DSL.literal("1234")).valueOf()); + + assertEquals(LITERAL_TRUE, DSL.json_function(DSL.literal("true")).valueOf()); + + assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal("null")).valueOf()); + + assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal("")).valueOf()); + + assertEquals( + ExprTupleValue.fromExprValueMap(Map.of()), DSL.json_function(DSL.literal("{}")).valueOf()); + } + + @Test + void json_returnsSemanticCheckException() { + // invalid type + assertThrows( + SemanticCheckException.class, () -> DSL.castJson(DSL.literal("invalid")).valueOf()); + + // missing bracket + assertThrows(SemanticCheckException.class, () -> DSL.castJson(DSL.literal("{{[}}")).valueOf()); + + // mnissing quote + assertThrows( + SemanticCheckException.class, () -> DSL.castJson(DSL.literal("\"missing quote")).valueOf()); + } } diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 0307fb4ca1..571cc94cc0 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -146,6 +146,7 @@ FLOAT: 'FLOAT'; STRING: 'STRING'; BOOLEAN: 'BOOLEAN'; IP: 'IP'; +JSON: 'JSON'; // SPECIAL CHARACTERS AND OPERATORS PIPE: '|'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 451edeb29b..d71c000b0e 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -972,4 +972,6 @@ keywordsCanBeId | SPARKLINE | C | DC + // JSON + | JSON ; From 80afe48b1bba00b4f51489fb6ff799187590ebfd Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Wed, 8 Jan 2025 10:56:50 -0800 Subject: [PATCH 14/29] PPL: Update json cast for review Signed-off-by: Andrew Carbonetto --- .../sql/expression/function/BuiltinFunctionName.java | 5 +---- ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 1 - ppl/src/main/antlr/OpenSearchPPLParser.g4 | 2 -- 3 files changed, 1 insertion(+), 7 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index ed54d30214..366321bed2 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -264,10 +264,7 @@ public enum BuiltinFunctionName { MULTIMATCH(FunctionName.of("multimatch")), MULTIMATCHQUERY(FunctionName.of("multimatchquery")), WILDCARDQUERY(FunctionName.of("wildcardquery")), - WILDCARD_QUERY(FunctionName.of("wildcard_query")), - - /* Json Functions. */ - JSON(FunctionName.of("json")); + WILDCARD_QUERY(FunctionName.of("wildcard_query")); private final FunctionName name; diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 571cc94cc0..0307fb4ca1 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -146,7 +146,6 @@ FLOAT: 'FLOAT'; STRING: 'STRING'; BOOLEAN: 'BOOLEAN'; IP: 'IP'; -JSON: 'JSON'; // SPECIAL CHARACTERS AND OPERATORS PIPE: '|'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index d71c000b0e..451edeb29b 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -972,6 +972,4 @@ keywordsCanBeId | SPARKLINE | C | DC - // JSON - | JSON ; From 7d212b83f148753d5bdcd83df823529621d8883f Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Wed, 8 Jan 2025 17:17:51 -0800 Subject: [PATCH 15/29] Fix testes Signed-off-by: Andrew Carbonetto --- .../org/opensearch/sql/expression/json/JsonFunctions.java | 6 ++++++ .../sql/expression/operator/convert/TypeCastOperators.java | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index 90f92e35cf..d0ebf9cce5 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -41,4 +41,10 @@ private DefaultFunctionResolver jsonFunction() { BuiltinFunctionName.JSON.getName(), impl(nullMissingHandling(JsonUtils::castJson), UNDEFINED, STRING)); } + + private DefaultFunctionResolver jsonFunction() { + return define( + BuiltinFunctionName.JSON.getName(), + impl(nullMissingHandling(JsonUtils::castJson), UNDEFINED, STRING)); + } } diff --git a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java index aaae000cda..c1391ac9ab 100644 --- a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java @@ -22,7 +22,6 @@ import static org.opensearch.sql.expression.function.FunctionDSL.implWithProperties; import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandling; import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandlingWithProperties; -import static org.opensearch.sql.utils.JsonUtils.castJson; import java.util.Arrays; import java.util.stream.Collectors; From 0e8bf0b4029f4654f0f7a46fdf56b7b65fcd20cc Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Mon, 13 Jan 2025 16:20:17 -0800 Subject: [PATCH 16/29] SPOTLESS Signed-off-by: Andrew Carbonetto --- .../expression/json/JsonFunctionsTest.java | 154 +++++++++--------- 1 file changed, 77 insertions(+), 77 deletions(-) diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index b138315eea..e5905c20f5 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -220,92 +220,92 @@ void json_returnsSemanticCheckException() { "Expected to throw SemanticCheckException when calling castJson with " + expr)); } - @Test - void json_returnsJsonObject() { - FunctionExpression exp; - - // Setup - final String objectJson = - "{\"foo\": \"foo\", \"fuzz\": true, \"bar\": 1234, \"bar2\": 12.34, \"baz\": null, " - + "\"obj\": {\"internal\": \"value\"}, \"arr\": [\"string\", true, null]}"; - - LinkedHashMap objectMap = new LinkedHashMap<>(); - objectMap.put("foo", new ExprStringValue("foo")); - objectMap.put("fuzz", ExprBooleanValue.of(true)); - objectMap.put("bar", new ExprLongValue(1234)); - objectMap.put("bar2", new ExprDoubleValue(12.34)); - objectMap.put("baz", ExprNullValue.of()); - objectMap.put( - "obj", ExprTupleValue.fromExprValueMap(Map.of("internal", new ExprStringValue("value")))); - objectMap.put( - "arr", - new ExprCollectionValue( - List.of(new ExprStringValue("string"), ExprBooleanValue.of(true), ExprNullValue.of()))); - ExprValue expectedTupleExpr = ExprTupleValue.fromExprValueMap(objectMap); - - // exercise - exp = DSL.json_function(DSL.literal(objectJson)); - - // Verify - var value = exp.valueOf(); - assertTrue(value instanceof ExprTupleValue); - assertEquals(expectedTupleExpr, value); - } - - @Test - void json_returnsJsonArray() { - FunctionExpression exp; - - // Setup - final String arrayJson = "[\"foo\", \"fuzz\", true, \"bar\", 1234, 12.34, null]"; - ExprValue expectedArrayExpr = - new ExprCollectionValue( - List.of( - new ExprStringValue("foo"), - new ExprStringValue("fuzz"), - LITERAL_TRUE, - new ExprStringValue("bar"), - new ExprIntegerValue(1234), - new ExprDoubleValue(12.34), - LITERAL_NULL)); + @Test + void json_returnsJsonObject() { + FunctionExpression exp; + + // Setup + final String objectJson = + "{\"foo\": \"foo\", \"fuzz\": true, \"bar\": 1234, \"bar2\": 12.34, \"baz\": null, " + + "\"obj\": {\"internal\": \"value\"}, \"arr\": [\"string\", true, null]}"; + + LinkedHashMap objectMap = new LinkedHashMap<>(); + objectMap.put("foo", new ExprStringValue("foo")); + objectMap.put("fuzz", ExprBooleanValue.of(true)); + objectMap.put("bar", new ExprLongValue(1234)); + objectMap.put("bar2", new ExprDoubleValue(12.34)); + objectMap.put("baz", ExprNullValue.of()); + objectMap.put( + "obj", ExprTupleValue.fromExprValueMap(Map.of("internal", new ExprStringValue("value")))); + objectMap.put( + "arr", + new ExprCollectionValue( + List.of(new ExprStringValue("string"), ExprBooleanValue.of(true), ExprNullValue.of()))); + ExprValue expectedTupleExpr = ExprTupleValue.fromExprValueMap(objectMap); + + // exercise + exp = DSL.json_function(DSL.literal(objectJson)); + + // Verify + var value = exp.valueOf(); + assertTrue(value instanceof ExprTupleValue); + assertEquals(expectedTupleExpr, value); + } + + @Test + void json_returnsJsonArray() { + FunctionExpression exp; + + // Setup + final String arrayJson = "[\"foo\", \"fuzz\", true, \"bar\", 1234, 12.34, null]"; + ExprValue expectedArrayExpr = + new ExprCollectionValue( + List.of( + new ExprStringValue("foo"), + new ExprStringValue("fuzz"), + LITERAL_TRUE, + new ExprStringValue("bar"), + new ExprIntegerValue(1234), + new ExprDoubleValue(12.34), + LITERAL_NULL)); - // exercise - exp = DSL.json_function(DSL.literal(arrayJson)); + // exercise + exp = DSL.json_function(DSL.literal(arrayJson)); - // Verify - var value = exp.valueOf(); - assertTrue(value instanceof ExprCollectionValue); - assertEquals(expectedArrayExpr, value); - } + // Verify + var value = exp.valueOf(); + assertTrue(value instanceof ExprCollectionValue); + assertEquals(expectedArrayExpr, value); + } - @Test - void json_returnsScalar() { - assertEquals( - new ExprStringValue("foobar"), DSL.json_function(DSL.literal("\"foobar\"")).valueOf()); + @Test + void json_returnsScalar() { + assertEquals( + new ExprStringValue("foobar"), DSL.json_function(DSL.literal("\"foobar\"")).valueOf()); - assertEquals(new ExprIntegerValue(1234), DSL.json_function(DSL.literal("1234")).valueOf()); + assertEquals(new ExprIntegerValue(1234), DSL.json_function(DSL.literal("1234")).valueOf()); - assertEquals(LITERAL_TRUE, DSL.json_function(DSL.literal("true")).valueOf()); + assertEquals(LITERAL_TRUE, DSL.json_function(DSL.literal("true")).valueOf()); - assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal("null")).valueOf()); + assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal("null")).valueOf()); - assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal("")).valueOf()); + assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal("")).valueOf()); - assertEquals( - ExprTupleValue.fromExprValueMap(Map.of()), DSL.json_function(DSL.literal("{}")).valueOf()); - } + assertEquals( + ExprTupleValue.fromExprValueMap(Map.of()), DSL.json_function(DSL.literal("{}")).valueOf()); + } - @Test - void json_returnsSemanticCheckException() { - // invalid type - assertThrows( - SemanticCheckException.class, () -> DSL.castJson(DSL.literal("invalid")).valueOf()); + @Test + void json_returnsSemanticCheckException() { + // invalid type + assertThrows( + SemanticCheckException.class, () -> DSL.castJson(DSL.literal("invalid")).valueOf()); - // missing bracket - assertThrows(SemanticCheckException.class, () -> DSL.castJson(DSL.literal("{{[}}")).valueOf()); + // missing bracket + assertThrows(SemanticCheckException.class, () -> DSL.castJson(DSL.literal("{{[}}")).valueOf()); - // mnissing quote - assertThrows( - SemanticCheckException.class, () -> DSL.castJson(DSL.literal("\"missing quote")).valueOf()); - } + // mnissing quote + assertThrows( + SemanticCheckException.class, () -> DSL.castJson(DSL.literal("\"missing quote")).valueOf()); + } } From bc1195178d2bfbb8eacd46f2d1759e962442fa2d Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Wed, 15 Jan 2025 10:33:11 -0800 Subject: [PATCH 17/29] Clean up for merge Signed-off-by: Andrew Carbonetto --- .../java/org/opensearch/sql/expression/json/JsonFunctions.java | 1 + .../org/opensearch/sql/expression/json/JsonFunctionsTest.java | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index d0ebf9cce5..3ff02571ef 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -16,6 +16,7 @@ import static org.opensearch.sql.data.type.ExprCoreType.UNDEFINED; import static org.opensearch.sql.expression.function.FunctionDSL.define; import static org.opensearch.sql.expression.function.FunctionDSL.impl; +import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandling; import lombok.experimental.UtilityClass; import org.opensearch.sql.expression.function.BuiltinFunctionName; diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index e5905c20f5..ed2bb9fd51 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -11,8 +11,6 @@ import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_FALSE; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_MISSING; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; -import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; -import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_TRUE; import java.util.LinkedHashMap; From d708a69e1bf3aa334f0135321c1074e6064c4c25 Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Thu, 16 Jan 2025 14:48:51 -0800 Subject: [PATCH 18/29] Add cast to scalar from undefined expression Signed-off-by: Andrew Carbonetto --- .../expression/operator/convert/TypeCastOperators.java | 5 ++--- .../expression/operator/convert/TypeCastOperatorTest.java | 8 ++++++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java index c1391ac9ab..1a43ef14e1 100644 --- a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java @@ -92,9 +92,8 @@ private static DefaultFunctionResolver castToByte() { STRING), impl(nullMissingHandling((v) -> new ExprByteValue(v.byteValue())), BYTE, DOUBLE), impl( - nullMissingHandling((v) -> new ExprByteValue(v.booleanValue() ? 1 : 0)), - BYTE, - BOOLEAN)); + nullMissingHandling((v) -> new ExprByteValue(v.booleanValue() ? 1 : 0)), BYTE, BOOLEAN), + impl(nullMissingHandling((v) -> v), BYTE, UNDEFINED)); } private static DefaultFunctionResolver castToShort() { diff --git a/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java b/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java index ff0c8bcc01..d588b8d45b 100644 --- a/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java @@ -354,6 +354,14 @@ void castUndefinedToBoolean() { assertEquals(ExprBooleanValue.of(true), expression.valueOf()); } + @Test + void castUndefinedToBoolean() { + // json cast is an UNDEFINED type expression + FunctionExpression expression = DSL.castBoolean(DSL.castJson(DSL.literal("true"))); + assertEquals(BOOLEAN, expression.type()); + assertEquals(ExprBooleanValue.of(true), expression.valueOf()); + } + @Test void castToDate() { FunctionExpression expression = DSL.castDate(DSL.literal("2012-08-07")); From c76a3faa1a438d4443859fbef8c57800c0ef1d7e Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Thu, 16 Jan 2025 15:34:17 -0800 Subject: [PATCH 19/29] Add test for missing/null Signed-off-by: Andrew Carbonetto --- .../opensearch/sql/expression/json/JsonFunctionsTest.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index ed2bb9fd51..61374948e8 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -283,10 +283,16 @@ void json_returnsScalar() { assertEquals(new ExprIntegerValue(1234), DSL.json_function(DSL.literal("1234")).valueOf()); + assertEquals(new ExprDoubleValue(12.34), DSL.json_function(DSL.literal("12.34")).valueOf()); + assertEquals(LITERAL_TRUE, DSL.json_function(DSL.literal("true")).valueOf()); assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal("null")).valueOf()); + assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal(LITERAL_NULL)).valueOf()); + + assertEquals(LITERAL_MISSING, DSL.json_function(DSL.literal(LITERAL_MISSING)).valueOf()); + assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal("")).valueOf()); assertEquals( From 26623c9bd165839cd777d0187b253d09b8a86953 Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Fri, 17 Jan 2025 09:58:42 -0800 Subject: [PATCH 20/29] Clean up merge conflicts Signed-off-by: Andrew Carbonetto --- .../sql/expression/json/JsonFunctions.java | 6 ---- .../operator/convert/TypeCastOperators.java | 5 +-- .../expression/json/JsonFunctionsTest.java | 32 +++++++++++-------- 3 files changed, 21 insertions(+), 22 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index 3ff02571ef..142bcbdf2e 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -5,12 +5,6 @@ package org.opensearch.sql.expression.json; -import com.fasterxml.jackson.databind.ObjectMapper; -import lombok.experimental.UtilityClass; -import org.opensearch.sql.expression.function.BuiltinFunctionName; -import org.opensearch.sql.expression.function.BuiltinFunctionRepository; -import org.opensearch.sql.expression.function.DefaultFunctionResolver; - import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.data.type.ExprCoreType.UNDEFINED; diff --git a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java index 1a43ef14e1..c1391ac9ab 100644 --- a/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/operator/convert/TypeCastOperators.java @@ -92,8 +92,9 @@ private static DefaultFunctionResolver castToByte() { STRING), impl(nullMissingHandling((v) -> new ExprByteValue(v.byteValue())), BYTE, DOUBLE), impl( - nullMissingHandling((v) -> new ExprByteValue(v.booleanValue() ? 1 : 0)), BYTE, BOOLEAN), - impl(nullMissingHandling((v) -> v), BYTE, UNDEFINED)); + nullMissingHandling((v) -> new ExprByteValue(v.booleanValue() ? 1 : 0)), + BYTE, + BOOLEAN)); } private static DefaultFunctionResolver castToShort() { diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index 61374948e8..a7fab75be6 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -242,12 +242,16 @@ void json_returnsJsonObject() { ExprValue expectedTupleExpr = ExprTupleValue.fromExprValueMap(objectMap); // exercise - exp = DSL.json_function(DSL.literal(objectJson)); + exp = DSL.stringToJson(DSL.literal(objectJson)); // Verify var value = exp.valueOf(); assertTrue(value instanceof ExprTupleValue); assertEquals(expectedTupleExpr, value); + + // also test the empty object case + assertEquals( + ExprTupleValue.fromExprValueMap(Map.of()), DSL.stringToJson(DSL.literal("{}")).valueOf()); } @Test @@ -268,35 +272,35 @@ void json_returnsJsonArray() { LITERAL_NULL)); // exercise - exp = DSL.json_function(DSL.literal(arrayJson)); + exp = DSL.stringToJson(DSL.literal(arrayJson)); // Verify var value = exp.valueOf(); assertTrue(value instanceof ExprCollectionValue); assertEquals(expectedArrayExpr, value); + + // also test the empty-array case + assertEquals(new ExprCollectionValue(List.of()), DSL.stringToJson(DSL.literal("[]")).valueOf()); } @Test void json_returnsScalar() { assertEquals( - new ExprStringValue("foobar"), DSL.json_function(DSL.literal("\"foobar\"")).valueOf()); - - assertEquals(new ExprIntegerValue(1234), DSL.json_function(DSL.literal("1234")).valueOf()); + new ExprStringValue("foobar"), DSL.stringToJson(DSL.literal("\"foobar\"")).valueOf()); - assertEquals(new ExprDoubleValue(12.34), DSL.json_function(DSL.literal("12.34")).valueOf()); + assertEquals(new ExprIntegerValue(1234), DSL.stringToJson(DSL.literal("1234")).valueOf()); - assertEquals(LITERAL_TRUE, DSL.json_function(DSL.literal("true")).valueOf()); + assertEquals(new ExprDoubleValue(12.34), DSL.stringToJson(DSL.literal("12.34")).valueOf()); - assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal("null")).valueOf()); + assertEquals(LITERAL_TRUE, DSL.stringToJson(DSL.literal("true")).valueOf()); - assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal(LITERAL_NULL)).valueOf()); + assertEquals(LITERAL_NULL, DSL.stringToJson(DSL.literal("null")).valueOf()); - assertEquals(LITERAL_MISSING, DSL.json_function(DSL.literal(LITERAL_MISSING)).valueOf()); + assertEquals(LITERAL_NULL, DSL.stringToJson(DSL.literal(LITERAL_NULL)).valueOf()); - assertEquals(LITERAL_NULL, DSL.json_function(DSL.literal("")).valueOf()); + assertEquals(LITERAL_MISSING, DSL.stringToJson(DSL.literal(LITERAL_MISSING)).valueOf()); - assertEquals( - ExprTupleValue.fromExprValueMap(Map.of()), DSL.json_function(DSL.literal("{}")).valueOf()); + assertEquals(LITERAL_NULL, DSL.stringToJson(DSL.literal("")).valueOf()); } @Test @@ -308,7 +312,7 @@ void json_returnsSemanticCheckException() { // missing bracket assertThrows(SemanticCheckException.class, () -> DSL.castJson(DSL.literal("{{[}}")).valueOf()); - // mnissing quote + // missing quote assertThrows( SemanticCheckException.class, () -> DSL.castJson(DSL.literal("\"missing quote")).valueOf()); } From 1af9053cb203995790794ab27b26bafcb2dad7ad Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Fri, 17 Jan 2025 15:24:03 -0800 Subject: [PATCH 21/29] Fix jacoco coverage Signed-off-by: Andrew Carbonetto --- .../org/opensearch/sql/expression/json/JsonFunctionsTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index a7fab75be6..3e74bbf0a8 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -293,6 +293,7 @@ void json_returnsScalar() { assertEquals(new ExprDoubleValue(12.34), DSL.stringToJson(DSL.literal("12.34")).valueOf()); assertEquals(LITERAL_TRUE, DSL.stringToJson(DSL.literal("true")).valueOf()); + assertEquals(LITERAL_FALSE, DSL.stringToJson(DSL.literal("false")).valueOf()); assertEquals(LITERAL_NULL, DSL.stringToJson(DSL.literal("null")).valueOf()); From 122cd1d82daceda77f2df95bfdbf1d99bb03972c Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Fri, 17 Jan 2025 15:44:23 -0800 Subject: [PATCH 22/29] Move to Switch by json type Signed-off-by: Andrew Carbonetto --- .../org/opensearch/sql/utils/JsonUtils.java | 28 ------------------- 1 file changed, 28 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java index 3c785f8b96..f38fe59789 100644 --- a/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java +++ b/core/src/main/java/org/opensearch/sql/utils/JsonUtils.java @@ -108,33 +108,5 @@ private static ExprValue processJsonNode(JsonNode jsonNode) { // in all other cases, return null return LITERAL_NULL; } - if (jsonNode.isIntegralNumber()) { - return new ExprIntegerValue(jsonNode.asLong()); - } - if (jsonNode.isBoolean()) { - return jsonNode.asBoolean() ? LITERAL_TRUE : LITERAL_FALSE; - } - if (jsonNode.isTextual()) { - return new ExprStringValue(jsonNode.asText()); - } - if (jsonNode.isArray()) { - List elements = new LinkedList<>(); - for (var iter = jsonNode.iterator(); iter.hasNext(); ) { - jsonNode = iter.next(); - elements.add(processJsonNode(jsonNode)); - } - return new ExprCollectionValue(elements); - } - if (jsonNode.isObject()) { - Map values = new LinkedHashMap<>(); - for (var iter = jsonNode.fields(); iter.hasNext(); ) { - Map.Entry entry = iter.next(); - values.put(entry.getKey(), processJsonNode(entry.getValue())); - } - return ExprTupleValue.fromExprValueMap(values); - } - - // in all other cases, return null - return LITERAL_NULL; } } From cbc05527f614e1aa653ea8ec87943033c70d2b20 Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Mon, 20 Jan 2025 10:02:13 -0800 Subject: [PATCH 23/29] Add JSON_OBJECT to ppl Signed-off-by: Andrew Carbonetto --- .../org/opensearch/sql/expression/DSL.java | 4 + .../function/BuiltinFunctionName.java | 1 + .../sql/expression/json/JsonFunctions.java | 76 +++++++++++++++++++ .../expression/json/JsonFunctionsTest.java | 76 +++++++++++++++++++ docs/user/ppl/functions/json.rst | 40 ++++++++++ .../opensearch/sql/ppl/JsonFunctionsIT.java | 36 ++++++++- ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 1 + ppl/src/main/antlr/OpenSearchPPLParser.g4 | 10 +++ 8 files changed, 241 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/DSL.java b/core/src/main/java/org/opensearch/sql/expression/DSL.java index 9246a31bb7..5c31326f4f 100644 --- a/core/src/main/java/org/opensearch/sql/expression/DSL.java +++ b/core/src/main/java/org/opensearch/sql/expression/DSL.java @@ -691,6 +691,10 @@ public static FunctionExpression stringToJson(Expression value) { return compile(FunctionProperties.None, BuiltinFunctionName.JSON, value); } + public static FunctionExpression jsonObject(Expression... expressions) { + return compile(FunctionProperties.None, BuiltinFunctionName.JSON_OBJECT, expressions); + } + public static Aggregator avg(Expression... expressions) { return aggregate(BuiltinFunctionName.AVG, expressions); } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index 366321bed2..56a1791d5e 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -207,6 +207,7 @@ public enum BuiltinFunctionName { /** Json Functions. */ JSON_VALID(FunctionName.of("json_valid")), JSON(FunctionName.of("json")), + JSON_OBJECT(FunctionName.of("json_object")), /** GEOSPATIAL Functions. */ GEOIP(FunctionName.of("geoip")), diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index 142bcbdf2e..d226390361 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -7,15 +7,33 @@ import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; import static org.opensearch.sql.data.type.ExprCoreType.STRING; +import static org.opensearch.sql.data.type.ExprCoreType.STRUCT; import static org.opensearch.sql.data.type.ExprCoreType.UNDEFINED; +import static org.opensearch.sql.expression.DSL.jsonObject; import static org.opensearch.sql.expression.function.FunctionDSL.define; import static org.opensearch.sql.expression.function.FunctionDSL.impl; import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandling; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; import lombok.experimental.UtilityClass; +import org.apache.commons.lang3.tuple.Pair; +import org.opensearch.sql.data.model.ExprTupleValue; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.type.ExprCoreType; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.exception.SemanticCheckException; +import org.opensearch.sql.expression.Expression; +import org.opensearch.sql.expression.FunctionExpression; +import org.opensearch.sql.expression.env.Environment; import org.opensearch.sql.expression.function.BuiltinFunctionName; import org.opensearch.sql.expression.function.BuiltinFunctionRepository; import org.opensearch.sql.expression.function.DefaultFunctionResolver; +import org.opensearch.sql.expression.function.FunctionBuilder; +import org.opensearch.sql.expression.function.FunctionName; +import org.opensearch.sql.expression.function.FunctionResolver; +import org.opensearch.sql.expression.function.FunctionSignature; import org.opensearch.sql.utils.JsonUtils; @UtilityClass @@ -23,6 +41,7 @@ public class JsonFunctions { public void register(BuiltinFunctionRepository repository) { repository.register(jsonValid()); repository.register(jsonFunction()); + repository.register(jsonObject()); } private DefaultFunctionResolver jsonValid() { @@ -42,4 +61,61 @@ private DefaultFunctionResolver jsonFunction() { BuiltinFunctionName.JSON.getName(), impl(nullMissingHandling(JsonUtils::castJson), UNDEFINED, STRING)); } + + /** Creates a JSON Object/tuple expr from a given list of kv pairs. */ + private static FunctionResolver jsonObject() { + return new FunctionResolver() { + @Override + public FunctionName getFunctionName() { + return BuiltinFunctionName.JSON_OBJECT.getName(); + } + + @Override + public Pair resolve( + FunctionSignature unresolvedSignature) { + List paramList = unresolvedSignature.getParamTypeList(); + // check that we got an even number of arguments + if (paramList.size() % 2 != 0) { + throw new SemanticCheckException( + String.format( + "Expected an even number of arguments but instead got %d arguments", + paramList.size())); + } + + // check that each "key" argument (of key-value pair) is a string + for (int i = 0; i < paramList.size(); i = i + 2) { + ExprType paramType = paramList.get(i); + if (!ExprCoreType.STRING.equals(paramType)) { + throw new SemanticCheckException( + String.format( + "Expected type %s instead of %s for parameter #%d", + ExprCoreType.STRING, paramType.typeName(), i + 1)); + } + } + + // return the unresolved signature and function builder + return Pair.of( + unresolvedSignature, + (functionProperties, arguments) -> + new FunctionExpression(getFunctionName(), arguments) { + @Override + public ExprValue valueOf(Environment valueEnv) { + LinkedHashMap tupleValues = new LinkedHashMap<>(); + Iterator iter = getArguments().iterator(); + while (iter.hasNext()) { + tupleValues.put( + iter.next().valueOf(valueEnv).stringValue(), + iter.next().valueOf(valueEnv)); + } + return ExprTupleValue.fromExprValueMap(tupleValues); + } + + @Override + public ExprType type() { + return STRUCT; + } + }); + } + }; + } } diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index 3e74bbf0a8..281cf1fae4 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -317,4 +317,80 @@ void json_returnsSemanticCheckException() { assertThrows( SemanticCheckException.class, () -> DSL.castJson(DSL.literal("\"missing quote")).valueOf()); } + + @Test + public void json_object_returns_tuple() { + FunctionExpression exp; + + // Setup + LinkedHashMap objectMap = new LinkedHashMap<>(); + objectMap.put("foo", new ExprStringValue("foo")); + objectMap.put("fuzz", ExprBooleanValue.of(true)); + objectMap.put("bar", new ExprLongValue(1234)); + objectMap.put("bar2", new ExprDoubleValue(12.34)); + objectMap.put("baz", ExprNullValue.of()); + objectMap.put( + "obj", ExprTupleValue.fromExprValueMap(Map.of("internal", new ExprStringValue("value")))); + // TODO: requires json_array() + // objectMap.put( + // "arr", + // new ExprCollectionValue( + // List.of(new ExprStringValue("string"), ExprBooleanValue.of(true), + // ExprNullValue.of()))); + ExprValue expectedTupleExpr = ExprTupleValue.fromExprValueMap(objectMap); + + // exercise + exp = + DSL.jsonObject( + DSL.literal("foo"), DSL.literal("foo"), + DSL.literal("fuzz"), DSL.literal(true), + DSL.literal("bar"), DSL.literal(1234), + DSL.literal("bar2"), DSL.literal(12.34), + DSL.literal("baz"), DSL.literal(LITERAL_NULL), + DSL.literal("obj"), DSL.jsonObject(DSL.literal("internal"), DSL.literal("value"))); + + // Verify + var value = exp.valueOf(); + assertTrue(value instanceof ExprTupleValue); + assertEquals(expectedTupleExpr, value); + } + + @Test + public void json_object_returns_empty_tuple() { + FunctionExpression exp; + + // Setup + LinkedHashMap objectMap = new LinkedHashMap<>(); + ExprValue expectedTupleExpr = ExprTupleValue.fromExprValueMap(objectMap); + + // exercise + exp = DSL.jsonObject(); + + // Verify + var value = exp.valueOf(); + assertTrue(value instanceof ExprTupleValue); + assertEquals(expectedTupleExpr, value); + } + + @Test + public void json_object_throws_SemanticCheckException() { + // wrong number of arguments + assertThrows( + SemanticCheckException.class, () -> DSL.jsonObject(DSL.literal("only one")).valueOf()); + assertThrows( + SemanticCheckException.class, + () -> + DSL.jsonObject(DSL.literal("one"), DSL.literal("two"), DSL.literal("three")).valueOf()); + + // key argument is not a string + assertThrows( + SemanticCheckException.class, + () -> DSL.jsonObject(DSL.literal(1234), DSL.literal("two")).valueOf()); + assertThrows( + SemanticCheckException.class, + () -> + DSL.jsonObject( + DSL.literal("one"), DSL.literal(true), DSL.literal(true), DSL.literal("four")) + .valueOf()); + } } diff --git a/docs/user/ppl/functions/json.rst b/docs/user/ppl/functions/json.rst index 77d9d00f45..1ce10df1b9 100644 --- a/docs/user/ppl/functions/json.rst +++ b/docs/user/ppl/functions/json.rst @@ -60,3 +60,43 @@ Example:: | json scalar string | "abc" | "abc" | | json empty string | | null | +---------------------+---------------------------------+-------------------------+ + +JSON_OBJECT +----------- + +Description +>>>>>>>>>>> + +Usage: `json_object(, [, , ]...)` returns a JSON object from key-value pairs. + +Argument type: +- A \ must be STRING. +- A \ can be a scalar, another json object, or json array type. Note: scalar fields will be treated as single-value. Use `json_array` to construct an array value from a multi-value. + +Return type: STRUCT + +Example: + + os> source=people | eval result = json_object('key', 123.45) | fields result + fetched rows / total rows = 1/1 + +------------------+ + | result | + +------------------+ + | {"key":123.45} | + +------------------+ + + os> source=people | eval result = json_object('outer', json_object('inner', 123.45)) | fields result + fetched rows / total rows = 1/1 + +------------------------------+ + | result | + +------------------------------+ + | {"outer":{"inner":123.45}} | + +------------------------------+ + + os> source=people | eval result = json_object('array_doc', json_array(123.45, "string", true, null)) | fields result + fetched rows / total rows = 1/1 + +------------------------------------------------+ + | result | + +------------------------------------------------+ + | {"array_doc":[123.45, "string", true, null]} | + +------------------------------------------------+ diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java index b6a2d5e4aa..05d7bf227a 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java @@ -175,12 +175,42 @@ public void test_cast_json_scalar_to_type() throws IOException { result = executeQuery( String.format( - "source=%s | where test_name='json scalar boolean true' OR test_name='json scalar" - + " boolean false' | eval casted=cast(json(json_string) as boolean) | fields" - + " test_name, casted", + "source=%s | " + + "where test_name='json scalar boolean true' OR test_name='json scalar boolean false' | " + + "eval casted=cast(json(json_string) as boolean) | " + + "fields test_name, casted", TEST_INDEX_JSON_TEST)); verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "boolean")); verifyDataRows( result, rows("json scalar boolean true", true), rows("json scalar boolean false", false)); } + + @Test + public void test_json_object() throws IOException { + JSONObject result; + + result = + executeQuery( + String.format( + "source=%s | where json_valid(json_string) | " + + "eval obj=json_object('key', json(json_string)) | " + + "fields test_name, obj", + TEST_INDEX_JSON_TEST)); + verifySchema(result, schema("test_name", null, "string"), schema("obj", null, "struct")); + verifyDataRows( + result, + rows( + "json nested object", + new JSONObject(Map.of("key", Map.of("a", "1", "b", Map.of("c", "3"), "d", List.of(1, 2, 3))))), + rows("json object", new JSONObject(Map.of("key", Map.of("a", "1", "b", "2")))), + rows("json array", new JSONObject(Map.of("key", List.of(1, 2, 3, 4)))), + rows("json scalar string", Map.of("key", "abc")), + rows("json scalar int", Map.of("key", 1234)), + rows("json scalar float", Map.of("key", 12.34)), + rows("json scalar double", Map.of("key", 2.99792458e8)), + rows("json scalar boolean true", Map.of("key", true)), + rows("json scalar boolean false", Map.of("key", false)), + rows("json empty string", Map.of()) + ); + } } diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 0307fb4ca1..9629500ba5 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -334,6 +334,7 @@ CIDRMATCH: 'CIDRMATCH'; // JSON FUNCTIONS JSON_VALID: 'JSON_VALID'; JSON: 'JSON'; +JSON_OBJECT: 'JSON_OBJECT'; // FLOWCONTROL FUNCTIONS IFNULL: 'IFNULL'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 451edeb29b..b840379f71 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -310,6 +310,7 @@ valueExpression | extractFunction # extractFunctionCall | getFormatFunction # getFormatFunctionCall | timestampFunction # timestampFunctionCall + | jsonObjectFunction # jsonObjectFunctionCall | LT_PRTHS valueExpression RT_PRTHS # parentheticValueExpr ; @@ -324,6 +325,10 @@ positionFunction : positionFunctionName LT_PRTHS functionArg IN functionArg RT_PRTHS ; +jsonObjectFunction + : jsonObjectFunctionName LT_PRTHS (functionArg COMMA functionArg (COMMA functionArg COMMA functionArg)*)? RT_PRTHS + ; + booleanExpression : booleanFunctionCall ; @@ -420,6 +425,7 @@ evalFunctionName | flowControlFunctionName | systemFunctionName | positionFunctionName + | jsonObjectFunctionName | jsonFunctionName | geoipFunctionName ; @@ -707,6 +713,10 @@ positionFunctionName : POSITION ; +jsonObjectFunctionName + : JSON_OBJECT + ; + jsonFunctionName : JSON ; From 423365bd78fb86f6fa1fca63d4a6c0211afb45eb Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Mon, 20 Jan 2025 10:47:17 -0800 Subject: [PATCH 24/29] SPOTLESS Signed-off-by: Andrew Carbonetto --- .../opensearch/sql/ppl/JsonFunctionsIT.java | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java index 05d7bf227a..c755b15a59 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java @@ -175,10 +175,9 @@ public void test_cast_json_scalar_to_type() throws IOException { result = executeQuery( String.format( - "source=%s | " + - "where test_name='json scalar boolean true' OR test_name='json scalar boolean false' | " + - "eval casted=cast(json(json_string) as boolean) | " + - "fields test_name, casted", + "source=%s | where test_name='json scalar boolean true' OR test_name='json scalar" + + " boolean false' | eval casted=cast(json(json_string) as boolean) | fields" + + " test_name, casted", TEST_INDEX_JSON_TEST)); verifySchema(result, schema("test_name", null, "string"), schema("casted", null, "boolean")); verifyDataRows( @@ -192,16 +191,17 @@ public void test_json_object() throws IOException { result = executeQuery( String.format( - "source=%s | where json_valid(json_string) | " + - "eval obj=json_object('key', json(json_string)) | " + - "fields test_name, obj", + "source=%s | where json_valid(json_string) | " + + "eval obj=json_object('key', json(json_string)) | " + + "fields test_name, obj", TEST_INDEX_JSON_TEST)); verifySchema(result, schema("test_name", null, "string"), schema("obj", null, "struct")); verifyDataRows( result, rows( "json nested object", - new JSONObject(Map.of("key", Map.of("a", "1", "b", Map.of("c", "3"), "d", List.of(1, 2, 3))))), + new JSONObject( + Map.of("key", Map.of("a", "1", "b", Map.of("c", "3"), "d", List.of(1, 2, 3))))), rows("json object", new JSONObject(Map.of("key", Map.of("a", "1", "b", "2")))), rows("json array", new JSONObject(Map.of("key", List.of(1, 2, 3, 4)))), rows("json scalar string", Map.of("key", "abc")), @@ -210,7 +210,6 @@ public void test_json_object() throws IOException { rows("json scalar double", Map.of("key", 2.99792458e8)), rows("json scalar boolean true", Map.of("key", true)), rows("json scalar boolean false", Map.of("key", false)), - rows("json empty string", Map.of()) - ); + rows("json empty string", Map.of())); } } From 09231fbe811f073dc2a07e0d57f7a3a4e108b3bf Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Tue, 21 Jan 2025 10:41:07 -0800 Subject: [PATCH 25/29] Fix doctest Signed-off-by: Andrew Carbonetto --- docs/user/ppl/functions/json.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/user/ppl/functions/json.rst b/docs/user/ppl/functions/json.rst index 1ce10df1b9..9cced28db3 100644 --- a/docs/user/ppl/functions/json.rst +++ b/docs/user/ppl/functions/json.rst @@ -79,21 +79,21 @@ Example: os> source=people | eval result = json_object('key', 123.45) | fields result fetched rows / total rows = 1/1 - +------------------+ - | result | - +------------------+ - | {"key":123.45} | - +------------------+ + +-----------------+ + | result | + |-----------------| + | {'key': 123.45} | + +-----------------+ os> source=people | eval result = json_object('outer', json_object('inner', 123.45)) | fields result fetched rows / total rows = 1/1 +------------------------------+ | result | - +------------------------------+ - | {"outer":{"inner":123.45}} | + |------------------------------| + | {'outer': {'inner': 123.45}} | +------------------------------+ - os> source=people | eval result = json_object('array_doc', json_array(123.45, "string", true, null)) | fields result + source=people | eval result = json_object('array_doc', json_array(123.45, "string", true, null)) | fields result fetched rows / total rows = 1/1 +------------------------------------------------+ | result | From 47593113a02881845592f857e1ffb767c2f99a3b Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Tue, 11 Feb 2025 22:36:49 -0800 Subject: [PATCH 26/29] Fix merge conflicts and tests Signed-off-by: Andrew Carbonetto --- .../sql/expression/json/JsonFunctions.java | 6 - .../expression/json/JsonFunctionsTest.java | 114 ------------------ .../convert/TypeCastOperatorTest.java | 8 -- 3 files changed, 128 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index d226390361..789462b6eb 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -56,12 +56,6 @@ private DefaultFunctionResolver jsonFunction() { impl(nullMissingHandling(JsonUtils::castJson), UNDEFINED, STRING)); } - private DefaultFunctionResolver jsonFunction() { - return define( - BuiltinFunctionName.JSON.getName(), - impl(nullMissingHandling(JsonUtils::castJson), UNDEFINED, STRING)); - } - /** Creates a JSON Object/tuple expr from a given list of kv pairs. */ private static FunctionResolver jsonObject() { return new FunctionResolver() { diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index 281cf1fae4..d4696f0a05 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -41,8 +41,6 @@ public class JsonFunctionsTest { public void json_valid_returns_false() { List expressions = List.of( - DSL.literal(LITERAL_MISSING), // missing returns false - DSL.literal(LITERAL_NULL), // null returns false DSL.literal("invalid"), // invalid type DSL.literal("{{[}}"), // missing bracket DSL.literal("[}"), // missing bracket @@ -192,118 +190,6 @@ void json_returnsScalar() { assertEquals(LITERAL_NULL, DSL.stringToJson(DSL.literal("")).valueOf()); } - @Test - void json_returnsSemanticCheckException() { - List expressions = - List.of( - DSL.literal("invalid"), // invalid type - DSL.literal("{{[}}"), // missing bracket - DSL.literal("[}"), // missing bracket - DSL.literal("}"), // missing bracket - DSL.literal("\"missing quote"), // missing quote - DSL.literal("abc"), // not a type - DSL.literal("97ab"), // not a type - DSL.literal("{1, 2, 3, 4}"), // invalid object - DSL.literal("{123: 1, true: 2, null: 3}"), // invalid object - DSL.literal("{\"invalid\":\"json\", \"string\"}"), // invalid object - DSL.literal("[\"a\": 1, \"b\": 2]") // invalid array - ); - - expressions.stream() - .forEach( - expr -> - assertThrows( - SemanticCheckException.class, - () -> DSL.castJson(expr).valueOf(), - "Expected to throw SemanticCheckException when calling castJson with " + expr)); - } - - @Test - void json_returnsJsonObject() { - FunctionExpression exp; - - // Setup - final String objectJson = - "{\"foo\": \"foo\", \"fuzz\": true, \"bar\": 1234, \"bar2\": 12.34, \"baz\": null, " - + "\"obj\": {\"internal\": \"value\"}, \"arr\": [\"string\", true, null]}"; - - LinkedHashMap objectMap = new LinkedHashMap<>(); - objectMap.put("foo", new ExprStringValue("foo")); - objectMap.put("fuzz", ExprBooleanValue.of(true)); - objectMap.put("bar", new ExprLongValue(1234)); - objectMap.put("bar2", new ExprDoubleValue(12.34)); - objectMap.put("baz", ExprNullValue.of()); - objectMap.put( - "obj", ExprTupleValue.fromExprValueMap(Map.of("internal", new ExprStringValue("value")))); - objectMap.put( - "arr", - new ExprCollectionValue( - List.of(new ExprStringValue("string"), ExprBooleanValue.of(true), ExprNullValue.of()))); - ExprValue expectedTupleExpr = ExprTupleValue.fromExprValueMap(objectMap); - - // exercise - exp = DSL.stringToJson(DSL.literal(objectJson)); - - // Verify - var value = exp.valueOf(); - assertTrue(value instanceof ExprTupleValue); - assertEquals(expectedTupleExpr, value); - - // also test the empty object case - assertEquals( - ExprTupleValue.fromExprValueMap(Map.of()), DSL.stringToJson(DSL.literal("{}")).valueOf()); - } - - @Test - void json_returnsJsonArray() { - FunctionExpression exp; - - // Setup - final String arrayJson = "[\"foo\", \"fuzz\", true, \"bar\", 1234, 12.34, null]"; - ExprValue expectedArrayExpr = - new ExprCollectionValue( - List.of( - new ExprStringValue("foo"), - new ExprStringValue("fuzz"), - LITERAL_TRUE, - new ExprStringValue("bar"), - new ExprIntegerValue(1234), - new ExprDoubleValue(12.34), - LITERAL_NULL)); - - // exercise - exp = DSL.stringToJson(DSL.literal(arrayJson)); - - // Verify - var value = exp.valueOf(); - assertTrue(value instanceof ExprCollectionValue); - assertEquals(expectedArrayExpr, value); - - // also test the empty-array case - assertEquals(new ExprCollectionValue(List.of()), DSL.stringToJson(DSL.literal("[]")).valueOf()); - } - - @Test - void json_returnsScalar() { - assertEquals( - new ExprStringValue("foobar"), DSL.stringToJson(DSL.literal("\"foobar\"")).valueOf()); - - assertEquals(new ExprIntegerValue(1234), DSL.stringToJson(DSL.literal("1234")).valueOf()); - - assertEquals(new ExprDoubleValue(12.34), DSL.stringToJson(DSL.literal("12.34")).valueOf()); - - assertEquals(LITERAL_TRUE, DSL.stringToJson(DSL.literal("true")).valueOf()); - assertEquals(LITERAL_FALSE, DSL.stringToJson(DSL.literal("false")).valueOf()); - - assertEquals(LITERAL_NULL, DSL.stringToJson(DSL.literal("null")).valueOf()); - - assertEquals(LITERAL_NULL, DSL.stringToJson(DSL.literal(LITERAL_NULL)).valueOf()); - - assertEquals(LITERAL_MISSING, DSL.stringToJson(DSL.literal(LITERAL_MISSING)).valueOf()); - - assertEquals(LITERAL_NULL, DSL.stringToJson(DSL.literal("")).valueOf()); - } - @Test void json_returnsSemanticCheckException() { // invalid type diff --git a/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java b/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java index d588b8d45b..ff0c8bcc01 100644 --- a/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/operator/convert/TypeCastOperatorTest.java @@ -354,14 +354,6 @@ void castUndefinedToBoolean() { assertEquals(ExprBooleanValue.of(true), expression.valueOf()); } - @Test - void castUndefinedToBoolean() { - // json cast is an UNDEFINED type expression - FunctionExpression expression = DSL.castBoolean(DSL.castJson(DSL.literal("true"))); - assertEquals(BOOLEAN, expression.type()); - assertEquals(ExprBooleanValue.of(true), expression.valueOf()); - } - @Test void castToDate() { FunctionExpression expression = DSL.castDate(DSL.literal("2012-08-07")); From 2dd3fc6bf715a6cc514001c4edbb119acd63601a Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Tue, 11 Feb 2025 22:42:54 -0800 Subject: [PATCH 27/29] Fix merge conflicts Signed-off-by: Andrew Carbonetto --- core/src/main/java/org/opensearch/sql/expression/DSL.java | 4 ---- .../org/opensearch/sql/expression/json/JsonFunctions.java | 3 +-- .../sql/expression/datetime/DateTimeFunctionTest.java | 2 -- .../org/opensearch/sql/expression/datetime/YearweekTest.java | 2 -- .../org/opensearch/sql/expression/json/JsonFunctionsTest.java | 2 ++ 5 files changed, 3 insertions(+), 10 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/expression/DSL.java b/core/src/main/java/org/opensearch/sql/expression/DSL.java index 5c31326f4f..950fdcef87 100644 --- a/core/src/main/java/org/opensearch/sql/expression/DSL.java +++ b/core/src/main/java/org/opensearch/sql/expression/DSL.java @@ -989,10 +989,6 @@ public static FunctionExpression geoip(Expression... args) { return compile(FunctionProperties.None, BuiltinFunctionName.GEOIP, args); } - public static FunctionExpression json_function(Expression value) { - return compile(FunctionProperties.None, BuiltinFunctionName.JSON, value); - } - @SuppressWarnings("unchecked") private static T compile( FunctionProperties functionProperties, BuiltinFunctionName bfn, Expression... args) { diff --git a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java index 789462b6eb..885380578c 100644 --- a/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java +++ b/core/src/main/java/org/opensearch/sql/expression/json/JsonFunctions.java @@ -46,8 +46,7 @@ public void register(BuiltinFunctionRepository repository) { private DefaultFunctionResolver jsonValid() { return define( - BuiltinFunctionName.JSON_VALID.getName(), - impl(nullMissingHandling(JsonUtils::isValidJson), BOOLEAN, STRING)); + BuiltinFunctionName.JSON_VALID.getName(), impl(JsonUtils::isValidJson, BOOLEAN, STRING)); } private DefaultFunctionResolver jsonFunction() { diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java index 115898e349..ad15dadfb7 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java @@ -1230,8 +1230,6 @@ public void testWeekFormats( expectedInteger); } - // subtracting 1 as a temporary fix for year 2024. - // Issue: https://github.com/opensearch-project/sql/issues/2477 @Test public void testWeekOfYearWithTimeType() { LocalDate today = LocalDate.now(functionProperties.getQueryStartClock()); diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java index ee4df23be4..d944f7c85c 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/YearweekTest.java @@ -99,8 +99,6 @@ public void testYearweekWithoutMode() { assertEquals(eval(expression), eval(expressionWithoutMode)); } - // subtracting 1 as a temporary fix for year 2024. - // Issue: https://github.com/opensearch-project/sql/issues/2477 @Test public void testYearweekWithTimeType() { int expected = getYearWeekBeforeSunday(LocalDate.now(functionProperties.getQueryStartClock())); diff --git a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java index d4696f0a05..c58e6f012b 100644 --- a/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/json/JsonFunctionsTest.java @@ -41,6 +41,8 @@ public class JsonFunctionsTest { public void json_valid_returns_false() { List expressions = List.of( + DSL.literal(LITERAL_MISSING), // missing returns false + DSL.literal(LITERAL_NULL), // null returns false DSL.literal("invalid"), // invalid type DSL.literal("{{[}}"), // missing bracket DSL.literal("[}"), // missing bracket From bb46eb9df862e9dcf89b9e8947756aeb9b179eed Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Wed, 12 Feb 2025 00:20:56 -0800 Subject: [PATCH 28/29] Fix IT tests Signed-off-by: Andrew Carbonetto --- .../src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java index c755b15a59..cdd4557adf 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java @@ -203,6 +203,9 @@ public void test_json_object() throws IOException { new JSONObject( Map.of("key", Map.of("a", "1", "b", Map.of("c", "3"), "d", List.of(1, 2, 3))))), rows("json object", new JSONObject(Map.of("key", Map.of("a", "1", "b", "2")))), + rows( + "json nested array", + new JSONObject(Map.of("key", List.of(1, 2, 3, Map.of("true", true, "number", 123))))), rows("json array", new JSONObject(Map.of("key", List.of(1, 2, 3, 4)))), rows("json scalar string", Map.of("key", "abc")), rows("json scalar int", Map.of("key", 1234)), From bc4ca9618b9b7b7aed51320f18a018ac0fa03242 Mon Sep 17 00:00:00 2001 From: Andrew Carbonetto Date: Wed, 12 Feb 2025 09:59:42 -0800 Subject: [PATCH 29/29] Fix IT tests Signed-off-by: Andrew Carbonetto --- .../src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java index cdd4557adf..87ff418dcb 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/JsonFunctionsIT.java @@ -201,7 +201,9 @@ public void test_json_object() throws IOException { rows( "json nested object", new JSONObject( - Map.of("key", Map.of("a", "1", "b", Map.of("c", "3"), "d", List.of(1, 2, 3))))), + Map.of( + "key", + Map.of("a", "1", "b", Map.of("c", "3"), "d", List.of(Boolean.FALSE, 3))))), rows("json object", new JSONObject(Map.of("key", Map.of("a", "1", "b", "2")))), rows( "json nested array",