Merge remote-tracking branch 'refs/remotes/origin/main' into feature/calcite-engine-nested

# Conflicts:
#	integ-test/src/test/java/org/opensearch/sql/ppl/ObjectFieldOperateIT.java

Author: qianheng-aws

build.gradle

@@ -52,7 +52,7 @@ buildscript {
         // TODO: Migrate following to Gradle version catalog || Read from OpenSearch BOM in the future.
         // See: https://github.com/opensearch-project/sql/issues/3257
         aws_java_sdk_version = "1.12.651"
-        guava_version = "32.1.3-jre"
+        guava_version = "33.3.0-jre"
         resilience4j_version = "1.5.0"
         hamcrest_version = "2.1"
         mockito_version = "5.7.0"

core/src/main/java/org/opensearch/sql/analysis/AnalysisContext.java

@@ -9,6 +9,7 @@
 import java.util.List;
 import java.util.Objects;
 import lombok.Getter;
+import org.opensearch.sql.executor.QueryType;
 import org.opensearch.sql.expression.NamedExpression;
 import org.opensearch.sql.expression.function.FunctionProperties;
 
@@ -21,8 +22,12 @@ public class AnalysisContext {
 
   @Getter private final FunctionProperties functionProperties;
 
+  public AnalysisContext(QueryType queryType) {
+    this(new TypeEnvironment(null), queryType);
+  }
+
   public AnalysisContext() {
-    this(new TypeEnvironment(null));
+    this(new TypeEnvironment(null), QueryType.SQL);
   }
 
   /**
@@ -31,9 +36,13 @@ public AnalysisContext() {
    * @param environment Env to set to a new instance.
    */
   public AnalysisContext(TypeEnvironment environment) {
+    this(environment, QueryType.SQL);
+  }
+
+  public AnalysisContext(TypeEnvironment environment, QueryType queryType) {
     this.environment = environment;
     this.namedParseExpressions = new ArrayList<>();
-    this.functionProperties = new FunctionProperties();
+    this.functionProperties = new FunctionProperties(queryType);
   }
 
   /** Push a new environment. */

core/src/main/java/org/opensearch/sql/calcite/udf/textUDF/LocateFunction.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.sql.calcite.udf.textUDF;
+
+import org.opensearch.sql.calcite.udf.UserDefinedFunction;
+
+public class LocateFunction implements UserDefinedFunction {
+  @Override
+  public Object eval(Object... args) {
+    if (args.length != 2 && args.length != 3) {
+      return new IllegalArgumentException(
+          "Invalid number of arguments, locate function expects 2 or 3 arguments");
+    }
+    String stringText = (String) args[0];
+    String targetText = (String) args[1];
+    if (stringText == null || targetText == null) {
+      return null;
+    }
+    if (args.length == 2) {
+      return stringText.indexOf(targetText) + 1;
+    } else {
+      int fromPosition = (int) args[2];
+      return stringText.indexOf(targetText, fromPosition - 1) + 1;
+    }
+  }
+}

core/src/main/java/org/opensearch/sql/calcite/udf/textUDF/ReplaceFunction.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.sql.calcite.udf.textUDF;
+
+import org.opensearch.sql.calcite.udf.UserDefinedFunction;
+
+/** We don't use calcite built in replace since it uses replace instead of replaceAll */
+public class ReplaceFunction implements UserDefinedFunction {
+  @Override
+  public Object eval(Object... args) {
+    if (args.length != 3) {
+      throw new IllegalArgumentException(
+          "replace Function requires 3 arguments, but current get: " + args.length);
+    }
+    for (int i = 0; i < args.length; i++) {
+      if (!(args[i] instanceof String)) {
+        throw new IllegalArgumentException(
+            "replace Function requires String arguments, but current get: " + args[i]);
+      }
+    }
+    String baseValue = (String) args[0];
+    String fromValue = (String) args[1];
+    String toValue = (String) args[2];
+    return baseValue.replace(fromValue, toValue);
+  }
+}

core/src/main/java/org/opensearch/sql/calcite/utils/BuiltinFunctionUtils.java

@@ -30,6 +30,8 @@
 import org.opensearch.sql.calcite.udf.mathUDF.EulerFunction;
 import org.opensearch.sql.calcite.udf.mathUDF.ModFunction;
 import org.opensearch.sql.calcite.udf.mathUDF.SqrtFunction;
+import org.opensearch.sql.calcite.udf.textUDF.LocateFunction;
+import org.opensearch.sql.calcite.udf.textUDF.ReplaceFunction;
 
 public interface BuiltinFunctionUtils {
 
@@ -66,6 +68,8 @@ static SqlOperator translate(String op) {
       case "/":
         return SqlStdOperatorTable.DIVIDE;
         // Built-in String Functions
+      case "ASCII":
+        return SqlStdOperatorTable.ASCII;
       case "CONCAT":
         return SqlLibraryOperators.CONCAT_FUNCTION;
       case "CONCAT_WS":
@@ -84,8 +88,19 @@ static SqlOperator translate(String op) {
         return SqlLibraryOperators.REVERSE;
       case "RIGHT":
         return SqlLibraryOperators.RIGHT;
-      case "SUBSTRING":
+      case "LEFT":
+        return SqlLibraryOperators.LEFT;
+      case "SUBSTRING", "SUBSTR":
         return SqlStdOperatorTable.SUBSTRING;
+      case "STRCMP":
+        return SqlLibraryOperators.STRCMP;
+      case "REPLACE":
+        return TransferUserDefinedFunction(ReplaceFunction.class, "REPLACE", ReturnTypes.CHAR);
+      case "LOCATE":
+        return TransferUserDefinedFunction(
+            LocateFunction.class,
+            "LOCATE",
+            getNullableReturnTypeInferenceForFixedType(SqlTypeName.INTEGER));
       case "UPPER":
         return SqlStdOperatorTable.UPPER;
         // Built-in Math Functions

core/src/main/java/org/opensearch/sql/calcite/utils/UserDefinedFunctionUtils.java

@@ -22,6 +22,7 @@
 import org.apache.calcite.sql.SqlOperator;
 import org.apache.calcite.sql.parser.SqlParserPos;
 import org.apache.calcite.sql.type.SqlReturnTypeInference;
+import org.apache.calcite.sql.type.SqlTypeName;
 import org.apache.calcite.sql.validate.SqlUserDefinedAggFunction;
 import org.apache.calcite.sql.validate.SqlUserDefinedFunction;
 import org.apache.calcite.tools.RelBuilder;
@@ -133,4 +134,25 @@ public static SqlReturnTypeInference getReturnTypeInference(int targetPosition)
           typeFactory.createSqlType(firstArgType.getSqlTypeName()), true);
     };
   }
+
+  /**
+   * For some udf/udaf, We need to create nullable types arguments.
+   *
+   * @param typeName
+   * @return a inference function
+   */
+  public static SqlReturnTypeInference getNullableReturnTypeInferenceForFixedType(
+      SqlTypeName typeName) {
+    return opBinding -> {
+      RelDataTypeFactory typeFactory = opBinding.getTypeFactory();
+
+      // Get argument types
+      List<RelDataType> argTypes = opBinding.collectOperandTypes();
+
+      if (argTypes.isEmpty()) {
+        throw new IllegalArgumentException("Function requires at least one argument.");
+      }
+      return typeFactory.createTypeWithNullability(typeFactory.createSqlType(typeName), true);
+    };
+  }
 }

core/src/main/java/org/opensearch/sql/executor/QueryService.java

@@ -80,7 +80,7 @@ public void execute(
       // TODO https://github.com/opensearch-project/sql/issues/3457
       // Calcite is not available for SQL query now. Maybe release in 3.1.0?
       if (!calciteEnabled || relNodeVisitor == null || queryType == QueryType.SQL) {
-        executePlan(analyze(plan), PlanContext.emptyPlanContext(), listener);
+        executePlan(analyze(plan, queryType), PlanContext.emptyPlanContext(), listener);
       } else {
         try {
           AccessController.doPrivileged(
@@ -106,7 +106,7 @@ public void execute(
             }
           }
           LOG.warn("Fallback to V2 query engine since got exception", t);
-          executePlan(analyze(plan), PlanContext.emptyPlanContext(), listener);
+          executePlan(analyze(plan, queryType), PlanContext.emptyPlanContext(), listener);
         }
       }
     } catch (Exception e) {
@@ -183,15 +183,15 @@ public void explain(
       QueryType queryType,
       ResponseListener<ExecutionEngine.ExplainResponse> listener) {
     try {
-      executionEngine.explain(plan(analyze(plan)), listener);
+      executionEngine.explain(plan(analyze(plan, queryType)), listener);
     } catch (Exception e) {
       listener.onFailure(e);
     }
   }
 
   /** Analyze {@link UnresolvedPlan}. */
-  public LogicalPlan analyze(UnresolvedPlan plan) {
-    return analyzer.analyze(plan, new AnalysisContext());
+  public LogicalPlan analyze(UnresolvedPlan plan, QueryType queryType) {
+    return analyzer.analyze(plan, new AnalysisContext(queryType));
   }
 
   public RelNode analyze(UnresolvedPlan plan, CalcitePlanContext context) {

core/src/main/java/org/opensearch/sql/executor/execution/AbstractPlan.java

@@ -22,7 +22,7 @@ public abstract class AbstractPlan {
   /** Uniq query id. */
   @Getter private final QueryId queryId;
 
-  @Getter private final QueryType queryType;
+  @Getter protected final QueryType queryType;
 
   /** Start query execution. */
   public abstract void execute();

core/src/main/java/org/opensearch/sql/executor/execution/StreamingQueryPlan.java

@@ -50,7 +50,7 @@ public StreamingQueryPlan(
   @Override
   public void execute() {
     try {
-      LogicalPlan logicalPlan = queryService.analyze(plan);
+      LogicalPlan logicalPlan = queryService.analyze(plan, queryType);
       StreamingSource streamingSource = buildStreamingSource(logicalPlan);
       streamingExecution =
           new MicroBatchStreamingExecution(

core/src/main/java/org/opensearch/sql/expression/function/FunctionProperties.java

@@ -10,19 +10,33 @@
 import java.time.Instant;
 import java.time.ZoneId;
 import lombok.EqualsAndHashCode;
-import lombok.RequiredArgsConstructor;
+import lombok.Getter;
+import org.opensearch.sql.executor.QueryType;
 
-@RequiredArgsConstructor
 @EqualsAndHashCode
 public class FunctionProperties implements Serializable {
 
   private final Instant nowInstant;
   private final ZoneId currentZoneId;
+  @Getter private final QueryType queryType;
 
   /** By default, use current time and current timezone. */
   public FunctionProperties() {
-    nowInstant = Instant.now();
-    currentZoneId = ZoneId.systemDefault();
+    this(QueryType.SQL);
+  }
+
+  public FunctionProperties(QueryType queryType) {
+    this(Instant.now(), ZoneId.systemDefault(), queryType);
+  }
+
+  public FunctionProperties(Instant nowInstant, ZoneId currentZoneId) {
+    this(nowInstant, currentZoneId, QueryType.SQL);
+  }
+
+  public FunctionProperties(Instant nowInstant, ZoneId currentZoneId, QueryType queryType) {
+    this.nowInstant = nowInstant;
+    this.currentZoneId = currentZoneId;
+    this.queryType = queryType;
   }
 
   /**

core/src/main/java/org/opensearch/sql/expression/system/SystemFunctions.java

@@ -6,12 +6,15 @@
 package org.opensearch.sql.expression.system;
 
 import static org.opensearch.sql.data.type.ExprCoreType.STRING;
+import static org.opensearch.sql.lang.PPLLangSpec.PPL_SPEC;
 
+import java.util.Locale;
 import lombok.experimental.UtilityClass;
 import org.apache.commons.lang3.tuple.Pair;
 import org.opensearch.sql.data.model.ExprStringValue;
 import org.opensearch.sql.data.model.ExprValue;
 import org.opensearch.sql.data.type.ExprType;
+import org.opensearch.sql.executor.QueryType;
 import org.opensearch.sql.expression.Expression;
 import org.opensearch.sql.expression.FunctionExpression;
 import org.opensearch.sql.expression.env.Environment;
@@ -41,7 +44,12 @@ public Pair<FunctionSignature, FunctionBuilder> resolve(
                 new FunctionExpression(BuiltinFunctionName.TYPEOF.getName(), arguments) {
                   @Override
                   public ExprValue valueOf(Environment<Expression, ExprValue> valueEnv) {
-                    return new ExprStringValue(getArguments().get(0).type().legacyTypeName());
+                    ExprType type = getArguments().get(0).type();
+                    return new ExprStringValue(
+                        (functionProperties.getQueryType() == QueryType.PPL
+                                ? PPL_SPEC.typeName(type)
+                                : type.legacyTypeName())
+                            .toUpperCase(Locale.ROOT));
                   }
 
                   @Override

core/src/main/java/org/opensearch/sql/lang/LangSpec.java

@@ -0,0 +1,62 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.sql.lang;
+
+import static org.opensearch.sql.executor.QueryType.PPL;
+
+import org.opensearch.sql.data.type.ExprType;
+import org.opensearch.sql.executor.QueryType;
+
+/**
+ * Represents a language specification for query processing.
+ *
+ * <p>This interface defines basic methods for language-specific behaviors, such as determining the
+ * language type and mapping expression types to type names. Two language specifications are
+ * provided: SQL and PPL.
+ */
+public interface LangSpec {
+
+  /** The default SQL language specification instance. */
+  LangSpec SQL_SPEC = new LangSpec() {};
+
+  /**
+   * Returns a language specification instance based on the provided language name.
+   *
+   * @param language the name of the language, case-insensitive.
+   * @return the PPL language specification if the language is PPL (ignoring case); otherwise, the
+   *     SQL language specification.
+   */
+  static LangSpec fromLanguage(String language) {
+    if (PPL.name().equalsIgnoreCase(language)) {
+      return PPLLangSpec.PPL_SPEC;
+    } else {
+      return SQL_SPEC;
+    }
+  }
+
+  /**
+   * Returns the language type of this specification.
+   *
+   * <p>By default, the language is considered SQL.
+   *
+   * @return the language type, SQL by default.
+   */
+  default QueryType language() {
+    return QueryType.SQL;
+  }
+
+  /**
+   * Returns the type name for the given expression type.
+   *
+   * <p>This default implementation returns the result of {@code exprType.typeName()}.
+   *
+   * @param exprType the expression type.
+   * @return the type name of the expression.
+   */
+  default String typeName(ExprType exprType) {
+    return exprType.typeName();
+  }
+}