Skip to content

Commit 60b8bd6

Browse files
qianheng-awsqianheng-aws
committed
Merge remote-tracking branch 'refs/remotes/origin/main' into feature/calcite-engine-nested
# Conflicts: # integ-test/src/test/java/org/opensearch/sql/ppl/ObjectFieldOperateIT.java
2 parents b8629d9 + 778a64d commit 60b8bd6

File tree

49 files changed

+849
-224
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

49 files changed

+849
-224
lines changed

‎build.gradle

+1-1
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ buildscript {
5252
// TODO: Migrate following to Gradle version catalog || Read from OpenSearch BOM in the future.
5353
// See: https://github.com/opensearch-project/sql/issues/3257
5454
aws_java_sdk_version = "1.12.651"
55-
guava_version = "32.1.3-jre"
55+
guava_version = "33.3.0-jre"
5656
resilience4j_version = "1.5.0"
5757
hamcrest_version = "2.1"
5858
mockito_version = "5.7.0"

‎core/src/main/java/org/opensearch/sql/analysis/AnalysisContext.java

+11-2
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
import java.util.List;
1010
import java.util.Objects;
1111
import lombok.Getter;
12+
import org.opensearch.sql.executor.QueryType;
1213
import org.opensearch.sql.expression.NamedExpression;
1314
import org.opensearch.sql.expression.function.FunctionProperties;
1415

@@ -21,8 +22,12 @@ public class AnalysisContext {
2122

2223
@Getter private final FunctionProperties functionProperties;
2324

25+
public AnalysisContext(QueryType queryType) {
26+
this(new TypeEnvironment(null), queryType);
27+
}
28+
2429
public AnalysisContext() {
25-
this(new TypeEnvironment(null));
30+
this(new TypeEnvironment(null), QueryType.SQL);
2631
}
2732

2833
/**
@@ -31,9 +36,13 @@ public AnalysisContext() {
3136
* @param environment Env to set to a new instance.
3237
*/
3338
public AnalysisContext(TypeEnvironment environment) {
39+
this(environment, QueryType.SQL);
40+
}
41+
42+
public AnalysisContext(TypeEnvironment environment, QueryType queryType) {
3443
this.environment = environment;
3544
this.namedParseExpressions = new ArrayList<>();
36-
this.functionProperties = new FunctionProperties();
45+
this.functionProperties = new FunctionProperties(queryType);
3746
}
3847

3948
/** Push a new environment. */

‎core/src/main/java/org/opensearch/sql/calcite/udf/textUDF/LocateFunction.java

+29
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package org.opensearch.sql.calcite.udf.textUDF;
7+
8+
import org.opensearch.sql.calcite.udf.UserDefinedFunction;
9+
10+
public class LocateFunction implements UserDefinedFunction {
11+
@Override
12+
public Object eval(Object... args) {
13+
if (args.length != 2 && args.length != 3) {
14+
return new IllegalArgumentException(
15+
"Invalid number of arguments, locate function expects 2 or 3 arguments");
16+
}
17+
String stringText = (String) args[0];
18+
String targetText = (String) args[1];
19+
if (stringText == null || targetText == null) {
20+
return null;
21+
}
22+
if (args.length == 2) {
23+
return stringText.indexOf(targetText) + 1;
24+
} else {
25+
int fromPosition = (int) args[2];
26+
return stringText.indexOf(targetText, fromPosition - 1) + 1;
27+
}
28+
}
29+
}

‎core/src/main/java/org/opensearch/sql/calcite/udf/textUDF/ReplaceFunction.java

+29
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package org.opensearch.sql.calcite.udf.textUDF;
7+
8+
import org.opensearch.sql.calcite.udf.UserDefinedFunction;
9+
10+
/** We don't use calcite built in replace since it uses replace instead of replaceAll */
11+
public class ReplaceFunction implements UserDefinedFunction {
12+
@Override
13+
public Object eval(Object... args) {
14+
if (args.length != 3) {
15+
throw new IllegalArgumentException(
16+
"replace Function requires 3 arguments, but current get: " + args.length);
17+
}
18+
for (int i = 0; i < args.length; i++) {
19+
if (!(args[i] instanceof String)) {
20+
throw new IllegalArgumentException(
21+
"replace Function requires String arguments, but current get: " + args[i]);
22+
}
23+
}
24+
String baseValue = (String) args[0];
25+
String fromValue = (String) args[1];
26+
String toValue = (String) args[2];
27+
return baseValue.replace(fromValue, toValue);
28+
}
29+
}

‎core/src/main/java/org/opensearch/sql/calcite/utils/BuiltinFunctionUtils.java

+16-1
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@
3030
import org.opensearch.sql.calcite.udf.mathUDF.EulerFunction;
3131
import org.opensearch.sql.calcite.udf.mathUDF.ModFunction;
3232
import org.opensearch.sql.calcite.udf.mathUDF.SqrtFunction;
33+
import org.opensearch.sql.calcite.udf.textUDF.LocateFunction;
34+
import org.opensearch.sql.calcite.udf.textUDF.ReplaceFunction;
3335

3436
public interface BuiltinFunctionUtils {
3537

@@ -66,6 +68,8 @@ static SqlOperator translate(String op) {
6668
case "/":
6769
return SqlStdOperatorTable.DIVIDE;
6870
// Built-in String Functions
71+
case "ASCII":
72+
return SqlStdOperatorTable.ASCII;
6973
case "CONCAT":
7074
return SqlLibraryOperators.CONCAT_FUNCTION;
7175
case "CONCAT_WS":
@@ -84,8 +88,19 @@ static SqlOperator translate(String op) {
8488
return SqlLibraryOperators.REVERSE;
8589
case "RIGHT":
8690
return SqlLibraryOperators.RIGHT;
87-
case "SUBSTRING":
91+
case "LEFT":
92+
return SqlLibraryOperators.LEFT;
93+
case "SUBSTRING", "SUBSTR":
8894
return SqlStdOperatorTable.SUBSTRING;
95+
case "STRCMP":
96+
return SqlLibraryOperators.STRCMP;
97+
case "REPLACE":
98+
return TransferUserDefinedFunction(ReplaceFunction.class, "REPLACE", ReturnTypes.CHAR);
99+
case "LOCATE":
100+
return TransferUserDefinedFunction(
101+
LocateFunction.class,
102+
"LOCATE",
103+
getNullableReturnTypeInferenceForFixedType(SqlTypeName.INTEGER));
89104
case "UPPER":
90105
return SqlStdOperatorTable.UPPER;
91106
// Built-in Math Functions

‎core/src/main/java/org/opensearch/sql/calcite/utils/UserDefinedFunctionUtils.java

+22
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import org.apache.calcite.sql.SqlOperator;
2323
import org.apache.calcite.sql.parser.SqlParserPos;
2424
import org.apache.calcite.sql.type.SqlReturnTypeInference;
25+
import org.apache.calcite.sql.type.SqlTypeName;
2526
import org.apache.calcite.sql.validate.SqlUserDefinedAggFunction;
2627
import org.apache.calcite.sql.validate.SqlUserDefinedFunction;
2728
import org.apache.calcite.tools.RelBuilder;
@@ -133,4 +134,25 @@ public static SqlReturnTypeInference getReturnTypeInference(int targetPosition)
133134
typeFactory.createSqlType(firstArgType.getSqlTypeName()), true);
134135
};
135136
}
137+
138+
/**
139+
* For some udf/udaf, We need to create nullable types arguments.
140+
*
141+
* @param typeName
142+
* @return a inference function
143+
*/
144+
public static SqlReturnTypeInference getNullableReturnTypeInferenceForFixedType(
145+
SqlTypeName typeName) {
146+
return opBinding -> {
147+
RelDataTypeFactory typeFactory = opBinding.getTypeFactory();
148+
149+
// Get argument types
150+
List<RelDataType> argTypes = opBinding.collectOperandTypes();
151+
152+
if (argTypes.isEmpty()) {
153+
throw new IllegalArgumentException("Function requires at least one argument.");
154+
}
155+
return typeFactory.createTypeWithNullability(typeFactory.createSqlType(typeName), true);
156+
};
157+
}
136158
}

‎core/src/main/java/org/opensearch/sql/executor/QueryService.java

+5-5
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ public void execute(
8080
// TODO https://github.com/opensearch-project/sql/issues/3457
8181
// Calcite is not available for SQL query now. Maybe release in 3.1.0?
8282
if (!calciteEnabled || relNodeVisitor == null || queryType == QueryType.SQL) {
83-
executePlan(analyze(plan), PlanContext.emptyPlanContext(), listener);
83+
executePlan(analyze(plan, queryType), PlanContext.emptyPlanContext(), listener);
8484
} else {
8585
try {
8686
AccessController.doPrivileged(
@@ -106,7 +106,7 @@ public void execute(
106106
}
107107
}
108108
LOG.warn("Fallback to V2 query engine since got exception", t);
109-
executePlan(analyze(plan), PlanContext.emptyPlanContext(), listener);
109+
executePlan(analyze(plan, queryType), PlanContext.emptyPlanContext(), listener);
110110
}
111111
}
112112
} catch (Exception e) {
@@ -183,15 +183,15 @@ public void explain(
183183
QueryType queryType,
184184
ResponseListener<ExecutionEngine.ExplainResponse> listener) {
185185
try {
186-
executionEngine.explain(plan(analyze(plan)), listener);
186+
executionEngine.explain(plan(analyze(plan, queryType)), listener);
187187
} catch (Exception e) {
188188
listener.onFailure(e);
189189
}
190190
}
191191

192192
/** Analyze {@link UnresolvedPlan}. */
193-
public LogicalPlan analyze(UnresolvedPlan plan) {
194-
return analyzer.analyze(plan, new AnalysisContext());
193+
public LogicalPlan analyze(UnresolvedPlan plan, QueryType queryType) {
194+
return analyzer.analyze(plan, new AnalysisContext(queryType));
195195
}
196196

197197
public RelNode analyze(UnresolvedPlan plan, CalcitePlanContext context) {

‎core/src/main/java/org/opensearch/sql/executor/execution/AbstractPlan.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ public abstract class AbstractPlan {
2222
/** Uniq query id. */
2323
@Getter private final QueryId queryId;
2424

25-
@Getter private final QueryType queryType;
25+
@Getter protected final QueryType queryType;
2626

2727
/** Start query execution. */
2828
public abstract void execute();

‎core/src/main/java/org/opensearch/sql/executor/execution/StreamingQueryPlan.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ public StreamingQueryPlan(
5050
@Override
5151
public void execute() {
5252
try {
53-
LogicalPlan logicalPlan = queryService.analyze(plan);
53+
LogicalPlan logicalPlan = queryService.analyze(plan, queryType);
5454
StreamingSource streamingSource = buildStreamingSource(logicalPlan);
5555
streamingExecution =
5656
new MicroBatchStreamingExecution(

‎core/src/main/java/org/opensearch/sql/expression/function/FunctionProperties.java

+18-4
Original file line numberDiff line numberDiff line change
@@ -10,19 +10,33 @@
1010
import java.time.Instant;
1111
import java.time.ZoneId;
1212
import lombok.EqualsAndHashCode;
13-
import lombok.RequiredArgsConstructor;
13+
import lombok.Getter;
14+
import org.opensearch.sql.executor.QueryType;
1415

15-
@RequiredArgsConstructor
1616
@EqualsAndHashCode
1717
public class FunctionProperties implements Serializable {
1818

1919
private final Instant nowInstant;
2020
private final ZoneId currentZoneId;
21+
@Getter private final QueryType queryType;
2122

2223
/** By default, use current time and current timezone. */
2324
public FunctionProperties() {
24-
nowInstant = Instant.now();
25-
currentZoneId = ZoneId.systemDefault();
25+
this(QueryType.SQL);
26+
}
27+
28+
public FunctionProperties(QueryType queryType) {
29+
this(Instant.now(), ZoneId.systemDefault(), queryType);
30+
}
31+
32+
public FunctionProperties(Instant nowInstant, ZoneId currentZoneId) {
33+
this(nowInstant, currentZoneId, QueryType.SQL);
34+
}
35+
36+
public FunctionProperties(Instant nowInstant, ZoneId currentZoneId, QueryType queryType) {
37+
this.nowInstant = nowInstant;
38+
this.currentZoneId = currentZoneId;
39+
this.queryType = queryType;
2640
}
2741

2842
/**

‎core/src/main/java/org/opensearch/sql/expression/system/SystemFunctions.java

+9-1
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,15 @@
66
package org.opensearch.sql.expression.system;
77

88
import static org.opensearch.sql.data.type.ExprCoreType.STRING;
9+
import static org.opensearch.sql.lang.PPLLangSpec.PPL_SPEC;
910

11+
import java.util.Locale;
1012
import lombok.experimental.UtilityClass;
1113
import org.apache.commons.lang3.tuple.Pair;
1214
import org.opensearch.sql.data.model.ExprStringValue;
1315
import org.opensearch.sql.data.model.ExprValue;
1416
import org.opensearch.sql.data.type.ExprType;
17+
import org.opensearch.sql.executor.QueryType;
1518
import org.opensearch.sql.expression.Expression;
1619
import org.opensearch.sql.expression.FunctionExpression;
1720
import org.opensearch.sql.expression.env.Environment;
@@ -41,7 +44,12 @@ public Pair<FunctionSignature, FunctionBuilder> resolve(
4144
new FunctionExpression(BuiltinFunctionName.TYPEOF.getName(), arguments) {
4245
@Override
4346
public ExprValue valueOf(Environment<Expression, ExprValue> valueEnv) {
44-
return new ExprStringValue(getArguments().get(0).type().legacyTypeName());
47+
ExprType type = getArguments().get(0).type();
48+
return new ExprStringValue(
49+
(functionProperties.getQueryType() == QueryType.PPL
50+
? PPL_SPEC.typeName(type)
51+
: type.legacyTypeName())
52+
.toUpperCase(Locale.ROOT));
4553
}
4654

4755
@Override

‎core/src/main/java/org/opensearch/sql/lang/LangSpec.java

+62
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package org.opensearch.sql.lang;
7+
8+
import static org.opensearch.sql.executor.QueryType.PPL;
9+
10+
import org.opensearch.sql.data.type.ExprType;
11+
import org.opensearch.sql.executor.QueryType;
12+
13+
/**
14+
* Represents a language specification for query processing.
15+
*
16+
* <p>This interface defines basic methods for language-specific behaviors, such as determining the
17+
* language type and mapping expression types to type names. Two language specifications are
18+
* provided: SQL and PPL.
19+
*/
20+
public interface LangSpec {
21+
22+
/** The default SQL language specification instance. */
23+
LangSpec SQL_SPEC = new LangSpec() {};
24+
25+
/**
26+
* Returns a language specification instance based on the provided language name.
27+
*
28+
* @param language the name of the language, case-insensitive.
29+
* @return the PPL language specification if the language is PPL (ignoring case); otherwise, the
30+
* SQL language specification.
31+
*/
32+
static LangSpec fromLanguage(String language) {
33+
if (PPL.name().equalsIgnoreCase(language)) {
34+
return PPLLangSpec.PPL_SPEC;
35+
} else {
36+
return SQL_SPEC;
37+
}
38+
}
39+
40+
/**
41+
* Returns the language type of this specification.
42+
*
43+
* <p>By default, the language is considered SQL.
44+
*
45+
* @return the language type, SQL by default.
46+
*/
47+
default QueryType language() {
48+
return QueryType.SQL;
49+
}
50+
51+
/**
52+
* Returns the type name for the given expression type.
53+
*
54+
* <p>This default implementation returns the result of {@code exprType.typeName()}.
55+
*
56+
* @param exprType the expression type.
57+
* @return the type name of the expression.
58+
*/
59+
default String typeName(ExprType exprType) {
60+
return exprType.typeName();
61+
}
62+
}

0 commit comments

Comments
 (0)