Support ppl BETWEEN operator within Calcite

core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java

@@ -40,21 +40,33 @@
 import org.opensearch.sql.ast.expression.Let;
 import org.opensearch.sql.ast.expression.UnresolvedExpression;
 import org.opensearch.sql.ast.expression.subquery.SubqueryExpression;
+import org.opensearch.sql.ast.tree.AD;
 import org.opensearch.sql.ast.tree.Aggregation;
+import org.opensearch.sql.ast.tree.CloseCursor;
 import org.opensearch.sql.ast.tree.Eval;
+import org.opensearch.sql.ast.tree.FetchCursor;
+import org.opensearch.sql.ast.tree.FillNull;
 import org.opensearch.sql.ast.tree.Filter;
 import org.opensearch.sql.ast.tree.Head;
 import org.opensearch.sql.ast.tree.Join;
+import org.opensearch.sql.ast.tree.Kmeans;
 import org.opensearch.sql.ast.tree.Lookup;
 import org.opensearch.sql.ast.tree.Lookup.OutputStrategy;
+import org.opensearch.sql.ast.tree.ML;
+import org.opensearch.sql.ast.tree.Paginate;
+import org.opensearch.sql.ast.tree.Parse;
 import org.opensearch.sql.ast.tree.Project;
+import org.opensearch.sql.ast.tree.RareTopN;
 import org.opensearch.sql.ast.tree.Relation;
 import org.opensearch.sql.ast.tree.Rename;
 import org.opensearch.sql.ast.tree.Sort;
 import org.opensearch.sql.ast.tree.Sort.SortOption;
 import org.opensearch.sql.ast.tree.SubqueryAlias;
+import org.opensearch.sql.ast.tree.TableFunction;
+import org.opensearch.sql.ast.tree.Trendline;
 import org.opensearch.sql.ast.tree.UnresolvedPlan;
 import org.opensearch.sql.calcite.utils.JoinAndLookupUtils;
+import org.opensearch.sql.exception.CalciteUnsupportedException;
 import org.opensearch.sql.exception.SemanticCheckException;
 
 public class CalciteRelNodeVisitor extends AbstractNodeVisitor<RelNode, CalcitePlanContext> {
@@ -447,4 +459,62 @@ public RelNode visitLookup(Lookup node, CalcitePlanContext context) {
 
     return context.relBuilder.peek();
   }
+
+  /*
+   * Unsupported Commands of PPL with Calcite for OpenSearch 3.0.0-beta
+   */
+  @Override
+  public RelNode visitAD(AD node, CalcitePlanContext context) {
+    throw new CalciteUnsupportedException("AD command is unsupported in Calcite");
+  }
+
+  @Override
+  public RelNode visitCloseCursor(CloseCursor closeCursor, CalcitePlanContext context) {
+    throw new CalciteUnsupportedException("Close cursor operation is unsupported in Calcite");
+  }
+
+  @Override
+  public RelNode visitFetchCursor(FetchCursor cursor, CalcitePlanContext context) {
+    throw new CalciteUnsupportedException("Fetch cursor operation is unsupported in Calcite");
+  }
+
+  @Override
+  public RelNode visitML(ML node, CalcitePlanContext context) {
+    throw new CalciteUnsupportedException("ML command is unsupported in Calcite");
+  }
+
+  @Override
+  public RelNode visitPaginate(Paginate paginate, CalcitePlanContext context) {
+    throw new CalciteUnsupportedException("Paginate operation is unsupported in Calcite");
+  }
+
+  @Override
+  public RelNode visitKmeans(Kmeans node, CalcitePlanContext context) {
+    throw new CalciteUnsupportedException("Kmeans command is unsupported in Calcite");
+  }
+
+  @Override
+  public RelNode visitFillNull(FillNull fillNull, CalcitePlanContext context) {
+    throw new CalciteUnsupportedException("FillNull command is unsupported in Calcite");
+  }
+
+  @Override
+  public RelNode visitParse(Parse node, CalcitePlanContext context) {
+    throw new CalciteUnsupportedException("Parse command is unsupported in Calcite");
+  }
+
+  @Override
+  public RelNode visitRareTopN(RareTopN node, CalcitePlanContext context) {
+    throw new CalciteUnsupportedException("Rare and Top commands are unsupported in Calcite");
+  }
+
+  @Override
+  public RelNode visitTableFunction(TableFunction node, CalcitePlanContext context) {
+    throw new CalciteUnsupportedException("Table function is unsupported in Calcite");
+  }
+
+  @Override
+  public RelNode visitTrendline(Trendline node, CalcitePlanContext context) {
+    throw new CalciteUnsupportedException("Trendline command is unsupported in Calcite");
+  }
 }

core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java

@@ -20,6 +20,7 @@
 import org.apache.calcite.rex.RexBuilder;
 import org.apache.calcite.rex.RexNode;
 import org.apache.calcite.sql.SqlIntervalQualifier;
+import org.apache.calcite.sql.SqlOperator;
 import org.apache.calcite.sql.fun.SqlStdOperatorTable;
 import org.apache.calcite.sql.parser.SqlParserUtil;
 import org.apache.calcite.sql.type.SqlTypeName;
@@ -29,6 +30,8 @@
 import org.opensearch.sql.ast.AbstractNodeVisitor;
 import org.opensearch.sql.ast.expression.Alias;
 import org.opensearch.sql.ast.expression.And;
+import org.opensearch.sql.ast.expression.Between;
+import org.opensearch.sql.ast.expression.Cast;
 import org.opensearch.sql.ast.expression.Compare;
 import org.opensearch.sql.ast.expression.EqualTo;
 import org.opensearch.sql.ast.expression.Function;
@@ -37,15 +40,19 @@
 import org.opensearch.sql.ast.expression.Not;
 import org.opensearch.sql.ast.expression.Or;
 import org.opensearch.sql.ast.expression.QualifiedName;
+import org.opensearch.sql.ast.expression.RelevanceFieldList;
 import org.opensearch.sql.ast.expression.Span;
 import org.opensearch.sql.ast.expression.SpanUnit;
 import org.opensearch.sql.ast.expression.UnresolvedExpression;
+import org.opensearch.sql.ast.expression.When;
 import org.opensearch.sql.ast.expression.Xor;
 import org.opensearch.sql.ast.expression.subquery.ExistsSubquery;
 import org.opensearch.sql.ast.expression.subquery.InSubquery;
 import org.opensearch.sql.ast.expression.subquery.ScalarSubquery;
 import org.opensearch.sql.ast.tree.UnresolvedPlan;
 import org.opensearch.sql.calcite.utils.BuiltinFunctionUtils;
+import org.opensearch.sql.common.utils.StringUtils;
+import org.opensearch.sql.exception.CalciteUnsupportedException;
 import org.opensearch.sql.exception.SemanticCheckException;
 
 @RequiredArgsConstructor
@@ -125,12 +132,9 @@ public RexNode visitOr(Or node, CalcitePlanContext context) {
 
   @Override
   public RexNode visitXor(Xor node, CalcitePlanContext context) {
-    final RelDataType booleanType =
-        context.rexBuilder.getTypeFactory().createSqlType(SqlTypeName.BOOLEAN);
     final RexNode left = analyze(node.getLeft(), context);
     final RexNode right = analyze(node.getRight(), context);
-    return context.rexBuilder.makeCall(
-        booleanType, SqlStdOperatorTable.BIT_XOR, List.of(left, right));
+    return context.relBuilder.notEquals(left, right);
   }
 
   @Override
@@ -141,12 +145,28 @@ public RexNode visitNot(Not node, CalcitePlanContext context) {
 
   @Override
   public RexNode visitCompare(Compare node, CalcitePlanContext context) {
-    final RelDataType booleanType =
-        context.rexBuilder.getTypeFactory().createSqlType(SqlTypeName.BOOLEAN);
+    SqlOperator op = BuiltinFunctionUtils.translate(node.getOperator());
     final RexNode left = analyze(node.getLeft(), context);
     final RexNode right = analyze(node.getRight(), context);
-    return context.rexBuilder.makeCall(
-        booleanType, BuiltinFunctionUtils.translate(node.getOperator()), List.of(left, right));
+    return context.relBuilder.call(op, left, right);
+  }
+
+  @Override
+  public RexNode visitBetween(Between node, CalcitePlanContext context) {
+    RexNode value = analyze(node.getValue(), context);
+    RexNode lowerBound = analyze(node.getLowerBound(), context);
+    RexNode upperBound = analyze(node.getUpperBound(), context);
+    RelDataType commonType = context.rexBuilder.commonType(value, lowerBound, upperBound);
+    if (commonType != null) {
+      lowerBound = context.rexBuilder.makeCast(commonType, lowerBound);
+      upperBound = context.rexBuilder.makeCast(commonType, upperBound);
+    } else {
+      throw new SemanticCheckException(
+          StringUtils.format(
+              "BETWEEN expression types are incompatible: [%s, %s, %s]",
+              value.getType(), lowerBound.getType(), upperBound.getType()));
+    }
+    return context.relBuilder.between(value, lowerBound, upperBound);
   }
 
   @Override
@@ -337,4 +357,22 @@ private RelNode resolveSubqueryPlan(UnresolvedPlan subquery, CalcitePlanContext
     }
     return subqueryRel;
   }
+
+  /*
+   * Unsupported Expressions of PPL with Calcite for OpenSearch 3.0.0-beta
+   */
+  @Override
+  public RexNode visitCast(Cast node, CalcitePlanContext context) {
+    throw new CalciteUnsupportedException("CastWhen function is unsupported in Calcite");
+  }
+
+  @Override
+  public RexNode visitWhen(When node, CalcitePlanContext context) {
+    throw new CalciteUnsupportedException("CastWhen function is unsupported in Calcite");
+  }
+
+  @Override
+  public RexNode visitRelevanceFieldList(RelevanceFieldList node, CalcitePlanContext context) {
+    throw new CalciteUnsupportedException("Relevance fields expression is unsupported in Calcite");
+  }
 }

core/src/main/java/org/opensearch/sql/calcite/ExtendedRexBuilder.java

@@ -5,6 +5,7 @@
 
 package org.opensearch.sql.calcite;
 
+import java.util.Arrays;
 import java.util.List;
 import org.apache.calcite.avatica.util.TimeUnit;
 import org.apache.calcite.rel.type.RelDataType;
@@ -35,6 +36,11 @@ public RexNode and(RexNode left, RexNode right) {
     return this.makeCall(booleanType, SqlStdOperatorTable.AND, List.of(left, right));
   }
 
+  public RelDataType commonType(RexNode... nodes) {
+    return this.getTypeFactory()
+        .leastRestrictive(Arrays.stream(nodes).map(RexNode::getType).toList());
+  }
+
   public SqlIntervalQualifier createIntervalUntil(SpanUnit unit) {
     TimeUnit timeUnit;
     switch (unit) {

core/src/main/java/org/opensearch/sql/calcite/utils/BuiltinFunctionUtils.java

@@ -34,12 +34,11 @@ static SqlOperator translate(String op) {
       case "NOT":
         return SqlStdOperatorTable.NOT;
       case "XOR":
-        return SqlStdOperatorTable.BIT_XOR;
+      case "!=":
+        return SqlStdOperatorTable.NOT_EQUALS;
       case "=":
         return SqlStdOperatorTable.EQUALS;
       case "<>":
-      case "!=":
-        return SqlStdOperatorTable.NOT_EQUALS;
       case ">":
         return SqlStdOperatorTable.GREATER_THAN;
       case ">=":
@@ -48,6 +47,8 @@ static SqlOperator translate(String op) {
         return SqlStdOperatorTable.LESS_THAN;
       case "<=":
         return SqlStdOperatorTable.LESS_THAN_OR_EQUAL;
+      case "REGEXP":
+        return SqlLibraryOperators.REGEXP;
       case "+":
         return SqlStdOperatorTable.PLUS;
       case "-":

core/src/main/java/org/opensearch/sql/exception/CalciteUnsupportedException.java

@@ -0,0 +1,13 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.sql.exception;
+
+public class CalciteUnsupportedException extends QueryEngineException {
+
+  public CalciteUnsupportedException(String message) {
+    super(message);
+  }
+}

core/src/main/java/org/opensearch/sql/executor/QueryService.java

@@ -92,7 +92,7 @@ public void execute(
             fallbackAllowed = settings.getSettingValue(Settings.Key.CALCITE_FALLBACK_ALLOWED);
           }
           if (!fallbackAllowed) {
-            throw e;
+            listener.onFailure(e);
           }
           LOG.warn("Fallback to V2 query engine since got exception", e);
           executePlan(analyze(plan), PlanContext.emptyPlanContext(), listener);

integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteOperatorIT.java

@@ -0,0 +1,26 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.sql.calcite.remote;
+
+import java.io.IOException;
+import org.junit.Ignore;
+import org.opensearch.sql.ppl.OperatorIT;
+
+public class CalciteOperatorIT extends OperatorIT {
+
+  @Override
+  public void init() throws IOException {
+    enableCalcite();
+    disallowCalciteFallback();
+    super.init();
+  }
+
+  @Ignore("https://github.com/opensearch-project/sql/issues/3398")
+  @Override
+  public void testModuleOperator() throws IOException {
+    super.testModuleOperator();
+  }
+}