diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java b/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java index 83e8b666ff..79a37e3bcf 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java @@ -19,6 +19,7 @@ import org.apache.calcite.tools.RelBuilder; import org.opensearch.sql.ast.expression.UnresolvedExpression; import org.opensearch.sql.calcite.utils.CalciteToolsHelper; +import org.opensearch.sql.executor.QueryType; public class CalcitePlanContext { @@ -26,13 +27,15 @@ public class CalcitePlanContext { public final Connection connection; public final RelBuilder relBuilder; public final ExtendedRexBuilder rexBuilder; + public final QueryType queryType; @Getter @Setter private boolean isResolvingJoinCondition = false; @Getter @Setter private boolean isResolvingExistsSubquery = false; private final Stack correlVar = new Stack<>(); - private CalcitePlanContext(FrameworkConfig config) { + private CalcitePlanContext(FrameworkConfig config, QueryType queryType) { this.config = config; + this.queryType = queryType; this.connection = CalciteToolsHelper.connect(config, TYPE_FACTORY); this.relBuilder = CalciteToolsHelper.create(config, TYPE_FACTORY, connection); this.rexBuilder = new ExtendedRexBuilder(relBuilder.getRexBuilder()); @@ -67,7 +70,7 @@ public Optional peekCorrelVar() { } } - public static CalcitePlanContext create(FrameworkConfig config) { - return new CalcitePlanContext(config); + public static CalcitePlanContext create(FrameworkConfig config, QueryType queryType) { + return new CalcitePlanContext(config, queryType); } } diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java index c0ececd56e..c4a8023ff5 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java @@ -8,6 +8,7 @@ import static org.opensearch.sql.ast.expression.SpanUnit.NONE; import static org.opensearch.sql.ast.expression.SpanUnit.UNKNOWN; import static org.opensearch.sql.calcite.utils.BuiltinFunctionUtils.translateArgument; +import static org.opensearch.sql.calcite.utils.PlanUtils.intervalUnitToSpanUnit; import java.math.BigDecimal; import java.util.List; @@ -19,6 +20,7 @@ import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlIntervalQualifier; import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.type.SqlTypeName; @@ -35,6 +37,7 @@ import org.opensearch.sql.ast.expression.EqualTo; import org.opensearch.sql.ast.expression.Function; import org.opensearch.sql.ast.expression.In; +import org.opensearch.sql.ast.expression.Interval; import org.opensearch.sql.ast.expression.Let; import org.opensearch.sql.ast.expression.Literal; import org.opensearch.sql.ast.expression.Not; @@ -116,6 +119,15 @@ public RexNode visitLiteral(Literal node, CalcitePlanContext context) { } } + @Override + public RexNode visitInterval(Interval node, CalcitePlanContext context) { + RexNode value = analyze(node.getValue(), context); + SqlIntervalQualifier intervalQualifier = + context.rexBuilder.createIntervalUntil(intervalUnitToSpanUnit(node.getUnit())); + return context.rexBuilder.makeIntervalLiteral( + new BigDecimal(value.toString()), intervalQualifier); + } + @Override public RexNode visitAnd(And node, CalcitePlanContext context) { final RelDataType booleanType = diff --git a/core/src/main/java/org/opensearch/sql/calcite/udf/systemUDF/TypeOfFunction.java b/core/src/main/java/org/opensearch/sql/calcite/udf/systemUDF/TypeOfFunction.java new file mode 100644 index 0000000000..0babd4cd1c --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/calcite/udf/systemUDF/TypeOfFunction.java @@ -0,0 +1,16 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.udf.systemUDF; + +import org.opensearch.sql.calcite.udf.UserDefinedFunction; + +public class TypeOfFunction implements UserDefinedFunction { + + @Override + public Object eval(Object... args) { + return args[0]; + } +} diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/BuiltinFunctionUtils.java b/core/src/main/java/org/opensearch/sql/calcite/utils/BuiltinFunctionUtils.java index c263cea098..1e845b9b82 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/BuiltinFunctionUtils.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/BuiltinFunctionUtils.java @@ -6,6 +6,7 @@ package org.opensearch.sql.calcite.utils; import static java.lang.Math.E; +import static org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.getLegacyTypeName; import static org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils.*; import java.math.BigDecimal; @@ -30,6 +31,7 @@ import org.opensearch.sql.calcite.udf.mathUDF.EulerFunction; import org.opensearch.sql.calcite.udf.mathUDF.ModFunction; import org.opensearch.sql.calcite.udf.mathUDF.SqrtFunction; +import org.opensearch.sql.calcite.udf.systemUDF.TypeOfFunction; import org.opensearch.sql.calcite.udf.textUDF.LocateFunction; import org.opensearch.sql.calcite.udf.textUDF.ReplaceFunction; @@ -197,6 +199,10 @@ static SqlOperator translate(String op) { return SqlStdOperatorTable.IS_NOT_NULL; case "IS NULL": return SqlStdOperatorTable.IS_NULL; + case "TYPEOF": + // TODO optimize this function to ImplementableFunction + return TransferUserDefinedFunction( + TypeOfFunction.class, "typeof", ReturnTypes.VARCHAR_2000_NULLABLE); // TODO Add more, ref RexImpTable default: throw new IllegalArgumentException("Unsupported operator: " + op); @@ -268,6 +274,11 @@ static List translateArgument( throw new IllegalArgumentException("Log cannot accept argument list: " + argList); } return LogArgs; + case "TYPEOF": + return List.of( + context.rexBuilder.makeLiteral( + getLegacyTypeName( + argList.getFirst().getType().getSqlTypeName(), context.queryType))); default: return argList; } diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchTypeFactory.java b/core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchTypeFactory.java index 1b2ffabdf1..8bc656d2ef 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchTypeFactory.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/OpenSearchTypeFactory.java @@ -21,10 +21,14 @@ import static org.opensearch.sql.data.type.ExprCoreType.TIME; import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; import static org.opensearch.sql.data.type.ExprCoreType.UNDEFINED; +import static org.opensearch.sql.data.type.ExprCoreType.UNKNOWN; +import static org.opensearch.sql.executor.QueryType.PPL; +import static org.opensearch.sql.lang.PPLLangSpec.PPL_SPEC; import java.lang.reflect.Type; import java.util.ArrayList; import java.util.List; +import java.util.Locale; import java.util.Map; import org.apache.calcite.jdbc.JavaTypeFactoryImpl; import org.apache.calcite.rel.type.RelDataType; @@ -35,6 +39,7 @@ import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.executor.OpenSearchTypeSystem; +import org.opensearch.sql.executor.QueryType; import org.opensearch.sql.storage.Table; /** This class is used to create RelDataType and map RelDataType to Java data type */ @@ -134,9 +139,13 @@ public static RelDataType convertExprTypeToRelDataType(ExprType fieldType, boole } } - /** Converts a Calcite data type to OpenSearch ExprCoreType. */ - public static ExprType convertRelDataTypeToExprType(RelDataType type) { - switch (type.getSqlTypeName()) { + /** + * Usually, {@link this#createSqlType(SqlTypeName, boolean)} is used to create RelDataType, then + * convert it to ExprType. This is a util to convert when you don't have typeFactory. So they are + * all ExprCoreType. + */ + public static ExprType convertSqlTypeNameToExprType(SqlTypeName sqlTypeName) { + switch (sqlTypeName) { case TINYINT: return BYTE; case SMALLINT: @@ -145,6 +154,7 @@ public static ExprType convertRelDataTypeToExprType(RelDataType type) { return INTEGER; case BIGINT: return LONG; + case FLOAT: case REAL: return FLOAT; case DOUBLE: @@ -157,16 +167,25 @@ public static ExprType convertRelDataTypeToExprType(RelDataType type) { case DATE: return DATE; case TIME: + case TIME_TZ: + case TIME_WITH_LOCAL_TIME_ZONE: return TIME; case TIMESTAMP: + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + case TIMESTAMP_TZ: return TIMESTAMP; - case GEOMETRY: - return IP; case INTERVAL_YEAR: + case INTERVAL_YEAR_MONTH: case INTERVAL_MONTH: case INTERVAL_DAY: + case INTERVAL_DAY_HOUR: + case INTERVAL_DAY_MINUTE: + case INTERVAL_DAY_SECOND: case INTERVAL_HOUR: + case INTERVAL_HOUR_MINUTE: + case INTERVAL_HOUR_SECOND: case INTERVAL_MINUTE: + case INTERVAL_MINUTE_SECOND: case INTERVAL_SECOND: return INTERVAL; case ARRAY: @@ -176,9 +195,33 @@ public static ExprType convertRelDataTypeToExprType(RelDataType type) { case NULL: return UNDEFINED; default: - throw new IllegalArgumentException( - "Unsupported conversion for Relational Data type: " + type.getSqlTypeName()); + return UNKNOWN; + } + } + + /** Get legacy name for a SqlTypeName. */ + public static String getLegacyTypeName(SqlTypeName sqlTypeName, QueryType queryType) { + switch (sqlTypeName) { + case BINARY: + case VARBINARY: + return "BINARY"; + case GEOMETRY: + return "GEO_POINT"; + default: + ExprType type = convertSqlTypeNameToExprType(sqlTypeName); + return (queryType == PPL ? PPL_SPEC.typeName(type) : type.legacyTypeName()) + .toUpperCase(Locale.ROOT); + } + } + + /** Converts a Calcite data type to OpenSearch ExprCoreType. */ + public static ExprType convertRelDataTypeToExprType(RelDataType type) { + ExprType exprType = convertSqlTypeNameToExprType(type.getSqlTypeName()); + if (exprType == UNKNOWN) { + throw new IllegalArgumentException( + "Unsupported conversion for Relational Data type: " + type.getSqlTypeName()); } + return exprType; } public static ExprValue getExprValueByExprType(ExprType type, Object value) { diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/PlanUtils.java b/core/src/main/java/org/opensearch/sql/calcite/utils/PlanUtils.java new file mode 100644 index 0000000000..32b17c5c99 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/PlanUtils.java @@ -0,0 +1,28 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.utils; + +import org.opensearch.sql.ast.expression.IntervalUnit; +import org.opensearch.sql.ast.expression.SpanUnit; + +public interface PlanUtils { + + static SpanUnit intervalUnitToSpanUnit(IntervalUnit unit) { + return switch (unit) { + case MICROSECOND -> SpanUnit.MILLISECOND; + case SECOND -> SpanUnit.SECOND; + case MINUTE -> SpanUnit.MINUTE; + case HOUR -> SpanUnit.HOUR; + case DAY -> SpanUnit.DAY; + case WEEK -> SpanUnit.WEEK; + case MONTH -> SpanUnit.MONTH; + case QUARTER -> SpanUnit.QUARTER; + case YEAR -> SpanUnit.YEAR; + case UNKNOWN -> SpanUnit.UNKNOWN; + default -> throw new UnsupportedOperationException("Unsupported interval unit: " + unit); + }; + } +} diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/UserDefinedFunctionUtils.java b/core/src/main/java/org/opensearch/sql/calcite/utils/UserDefinedFunctionUtils.java index 7a722b74c3..519edd36eb 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/UserDefinedFunctionUtils.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/UserDefinedFunctionUtils.java @@ -21,6 +21,7 @@ import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlOperator; import org.apache.calcite.sql.parser.SqlParserPos; +import org.apache.calcite.sql.type.InferTypes; import org.apache.calcite.sql.type.SqlReturnTypeInference; import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.sql.validate.SqlUserDefinedAggFunction; @@ -63,7 +64,12 @@ public static SqlOperator TransferUserDefinedFunction( SqlIdentifier udfLtrimIdentifier = new SqlIdentifier(Collections.singletonList(functionName), null, SqlParserPos.ZERO, null); return new SqlUserDefinedFunction( - udfLtrimIdentifier, SqlKind.OTHER_FUNCTION, returnType, null, null, udfFunction); + udfLtrimIdentifier, + SqlKind.OTHER_FUNCTION, + returnType, + InferTypes.ANY_NULLABLE, + null, + udfFunction); } public static SqlReturnTypeInference getReturnTypeInferenceForArray() { diff --git a/core/src/main/java/org/opensearch/sql/executor/QueryService.java b/core/src/main/java/org/opensearch/sql/executor/QueryService.java index b645614ff8..900504a055 100644 --- a/core/src/main/java/org/opensearch/sql/executor/QueryService.java +++ b/core/src/main/java/org/opensearch/sql/executor/QueryService.java @@ -87,7 +87,7 @@ public void execute( (PrivilegedAction) () -> { final FrameworkConfig config = buildFrameworkConfig(); - final CalcitePlanContext context = CalcitePlanContext.create(config); + final CalcitePlanContext context = CalcitePlanContext.create(config, queryType); executePlanByCalcite(analyze(plan, context), context, listener); return null; }); diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/nonfallback/NonFallbackCalciteSystemFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/nonfallback/NonFallbackCalciteSystemFunctionIT.java index bfb8afa87f..862950f455 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/nonfallback/NonFallbackCalciteSystemFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/nonfallback/NonFallbackCalciteSystemFunctionIT.java @@ -5,10 +5,8 @@ package org.opensearch.sql.calcite.remote.nonfallback; -import org.junit.Ignore; import org.opensearch.sql.calcite.remote.fallback.CalciteSystemFunctionIT; -@Ignore("https://github.com/opensearch-project/sql/issues/3418") public class NonFallbackCalciteSystemFunctionIT extends CalciteSystemFunctionIT { @Override public void init() throws Exception { diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLBuiltinFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLBuiltinFunctionIT.java index b33551b1d8..7a81d12492 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLBuiltinFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLBuiltinFunctionIT.java @@ -17,6 +17,7 @@ import java.io.IOException; import org.json.JSONObject; +import org.junit.Ignore; import org.junit.jupiter.api.Test; public class CalcitePPLBuiltinFunctionIT extends CalcitePPLIntegTestCase { @@ -106,6 +107,43 @@ public void testAtanAndAtan2WithSort() { verifyDataRowsInOrder(actual, rows("Hello", 30, 4), rows("Jake", 70, 4)); } + @Test + public void testTypeOfBasic() { + JSONObject result = + executeQuery( + String.format( + """ + source=%s + | eval `typeof(1)` = typeof(1) + | eval `typeof(true)` = typeof(true) + | eval `typeof(2.0)` = typeof(2.0) + | eval `typeof("2.0")` = typeof("2.0") + | eval `typeof(name)` = typeof(name) + | eval `typeof(country)` = typeof(country) + | eval `typeof(age)` = typeof(age) + | eval `typeof(interval)` = typeof(INTERVAL 2 DAY) + | fields `typeof(1)`, `typeof(true)`, `typeof(2.0)`, `typeof("2.0")`, `typeof(name)`, `typeof(country)`, `typeof(age)`, `typeof(interval)` + | head 1 + """, + TEST_INDEX_STATE_COUNTRY)); + verifyDataRows( + result, rows("INT", "BOOLEAN", "DOUBLE", "STRING", "STRING", "STRING", "INT", "INTERVAL")); + } + + @Ignore("https://github.com/opensearch-project/sql/issues/3400") + public void testTypeOfDateTime() { + JSONObject result = + executeQuery( + String.format( + """ + source=%s + | eval `typeof(date)` = typeof(DATE('2008-04-14')) + | eval `typeof(now())` = typeof(now()) + | fields `typeof(date)`, `typeof(now())` + """, + TEST_INDEX_STATE_COUNTRY)); + } + @Test public void testCeilingAndFloor() { JSONObject actual = diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java index b0e119bffb..139b69a988 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java @@ -82,6 +82,8 @@ public void typeof_opensearch_types() throws IOException { + " | fields `text`, `date`, `date_nanos`, `boolean`, `object`, `keyword`," + " `ip`, `binary`, `geo_point`", TEST_INDEX_DATATYPE_NONNUMERIC)); + // TODO https://github.com/opensearch-project/sql/issues/3322 + // TO support IP, we need support UDT. verifyDataRows( response, rows( @@ -91,7 +93,7 @@ public void typeof_opensearch_types() throws IOException { "BOOLEAN", "STRUCT", "STRING", - "IP", + isCalciteEnabled() ? "STRING" : "IP", "BINARY", "GEO_POINT")); } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAbstractTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAbstractTest.java index 9bacbf07a6..91eb9279e1 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAbstractTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAbstractTest.java @@ -10,6 +10,7 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; +import static org.opensearch.sql.executor.QueryType.PPL; import java.sql.PreparedStatement; import java.sql.SQLException; @@ -72,7 +73,7 @@ protected CalcitePlanContext createBuilderContext() { /** Creates a CalcitePlanContext with transformed config. */ private CalcitePlanContext createBuilderContext(UnaryOperator transform) { config.context(Contexts.of(transform.apply(RelBuilder.Config.DEFAULT))); - return CalcitePlanContext.create(config.build()); + return CalcitePlanContext.create(config.build(), PPL); } /** Get the root RelNode of the given PPL query */