Skip to content

Commit 1cefc8c

Browse files
LantaoJinLantaoJin
authored
Support ppl BETWEEN operator within Calcite (#3433)
* Support ppl BETWEEN operation within Calcite Signed-off-by: Lantao Jin <[email protected]> * add more tests Signed-off-by: Lantao Jin <[email protected]> --------- Signed-off-by: Lantao Jin <[email protected]>
1 parent 7dd2811 commit 1cefc8c

File tree

14 files changed

+370
-14
lines changed

14 files changed

+370
-14
lines changed

‎core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java

+70
Original file line numberDiff line numberDiff line change
@@ -40,21 +40,33 @@
4040
import org.opensearch.sql.ast.expression.Let;
4141
import org.opensearch.sql.ast.expression.UnresolvedExpression;
4242
import org.opensearch.sql.ast.expression.subquery.SubqueryExpression;
43+
import org.opensearch.sql.ast.tree.AD;
4344
import org.opensearch.sql.ast.tree.Aggregation;
45+
import org.opensearch.sql.ast.tree.CloseCursor;
4446
import org.opensearch.sql.ast.tree.Eval;
47+
import org.opensearch.sql.ast.tree.FetchCursor;
48+
import org.opensearch.sql.ast.tree.FillNull;
4549
import org.opensearch.sql.ast.tree.Filter;
4650
import org.opensearch.sql.ast.tree.Head;
4751
import org.opensearch.sql.ast.tree.Join;
52+
import org.opensearch.sql.ast.tree.Kmeans;
4853
import org.opensearch.sql.ast.tree.Lookup;
4954
import org.opensearch.sql.ast.tree.Lookup.OutputStrategy;
55+
import org.opensearch.sql.ast.tree.ML;
56+
import org.opensearch.sql.ast.tree.Paginate;
57+
import org.opensearch.sql.ast.tree.Parse;
5058
import org.opensearch.sql.ast.tree.Project;
59+
import org.opensearch.sql.ast.tree.RareTopN;
5160
import org.opensearch.sql.ast.tree.Relation;
5261
import org.opensearch.sql.ast.tree.Rename;
5362
import org.opensearch.sql.ast.tree.Sort;
5463
import org.opensearch.sql.ast.tree.Sort.SortOption;
5564
import org.opensearch.sql.ast.tree.SubqueryAlias;
65+
import org.opensearch.sql.ast.tree.TableFunction;
66+
import org.opensearch.sql.ast.tree.Trendline;
5667
import org.opensearch.sql.ast.tree.UnresolvedPlan;
5768
import org.opensearch.sql.calcite.utils.JoinAndLookupUtils;
69+
import org.opensearch.sql.exception.CalciteUnsupportedException;
5870
import org.opensearch.sql.exception.SemanticCheckException;
5971

6072
public class CalciteRelNodeVisitor extends AbstractNodeVisitor<RelNode, CalcitePlanContext> {
@@ -447,4 +459,62 @@ public RelNode visitLookup(Lookup node, CalcitePlanContext context) {
447459

448460
return context.relBuilder.peek();
449461
}
462+
463+
/*
464+
* Unsupported Commands of PPL with Calcite for OpenSearch 3.0.0-beta
465+
*/
466+
@Override
467+
public RelNode visitAD(AD node, CalcitePlanContext context) {
468+
throw new CalciteUnsupportedException("AD command is unsupported in Calcite");
469+
}
470+
471+
@Override
472+
public RelNode visitCloseCursor(CloseCursor closeCursor, CalcitePlanContext context) {
473+
throw new CalciteUnsupportedException("Close cursor operation is unsupported in Calcite");
474+
}
475+
476+
@Override
477+
public RelNode visitFetchCursor(FetchCursor cursor, CalcitePlanContext context) {
478+
throw new CalciteUnsupportedException("Fetch cursor operation is unsupported in Calcite");
479+
}
480+
481+
@Override
482+
public RelNode visitML(ML node, CalcitePlanContext context) {
483+
throw new CalciteUnsupportedException("ML command is unsupported in Calcite");
484+
}
485+
486+
@Override
487+
public RelNode visitPaginate(Paginate paginate, CalcitePlanContext context) {
488+
throw new CalciteUnsupportedException("Paginate operation is unsupported in Calcite");
489+
}
490+
491+
@Override
492+
public RelNode visitKmeans(Kmeans node, CalcitePlanContext context) {
493+
throw new CalciteUnsupportedException("Kmeans command is unsupported in Calcite");
494+
}
495+
496+
@Override
497+
public RelNode visitFillNull(FillNull fillNull, CalcitePlanContext context) {
498+
throw new CalciteUnsupportedException("FillNull command is unsupported in Calcite");
499+
}
500+
501+
@Override
502+
public RelNode visitParse(Parse node, CalcitePlanContext context) {
503+
throw new CalciteUnsupportedException("Parse command is unsupported in Calcite");
504+
}
505+
506+
@Override
507+
public RelNode visitRareTopN(RareTopN node, CalcitePlanContext context) {
508+
throw new CalciteUnsupportedException("Rare and Top commands are unsupported in Calcite");
509+
}
510+
511+
@Override
512+
public RelNode visitTableFunction(TableFunction node, CalcitePlanContext context) {
513+
throw new CalciteUnsupportedException("Table function is unsupported in Calcite");
514+
}
515+
516+
@Override
517+
public RelNode visitTrendline(Trendline node, CalcitePlanContext context) {
518+
throw new CalciteUnsupportedException("Trendline command is unsupported in Calcite");
519+
}
450520
}

‎core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java

+46-8
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import org.apache.calcite.rex.RexBuilder;
2121
import org.apache.calcite.rex.RexNode;
2222
import org.apache.calcite.sql.SqlIntervalQualifier;
23+
import org.apache.calcite.sql.SqlOperator;
2324
import org.apache.calcite.sql.fun.SqlStdOperatorTable;
2425
import org.apache.calcite.sql.parser.SqlParserUtil;
2526
import org.apache.calcite.sql.type.SqlTypeName;
@@ -29,6 +30,8 @@
2930
import org.opensearch.sql.ast.AbstractNodeVisitor;
3031
import org.opensearch.sql.ast.expression.Alias;
3132
import org.opensearch.sql.ast.expression.And;
33+
import org.opensearch.sql.ast.expression.Between;
34+
import org.opensearch.sql.ast.expression.Cast;
3235
import org.opensearch.sql.ast.expression.Compare;
3336
import org.opensearch.sql.ast.expression.EqualTo;
3437
import org.opensearch.sql.ast.expression.Function;
@@ -37,15 +40,19 @@
3740
import org.opensearch.sql.ast.expression.Not;
3841
import org.opensearch.sql.ast.expression.Or;
3942
import org.opensearch.sql.ast.expression.QualifiedName;
43+
import org.opensearch.sql.ast.expression.RelevanceFieldList;
4044
import org.opensearch.sql.ast.expression.Span;
4145
import org.opensearch.sql.ast.expression.SpanUnit;
4246
import org.opensearch.sql.ast.expression.UnresolvedExpression;
47+
import org.opensearch.sql.ast.expression.When;
4348
import org.opensearch.sql.ast.expression.Xor;
4449
import org.opensearch.sql.ast.expression.subquery.ExistsSubquery;
4550
import org.opensearch.sql.ast.expression.subquery.InSubquery;
4651
import org.opensearch.sql.ast.expression.subquery.ScalarSubquery;
4752
import org.opensearch.sql.ast.tree.UnresolvedPlan;
4853
import org.opensearch.sql.calcite.utils.BuiltinFunctionUtils;
54+
import org.opensearch.sql.common.utils.StringUtils;
55+
import org.opensearch.sql.exception.CalciteUnsupportedException;
4956
import org.opensearch.sql.exception.SemanticCheckException;
5057

5158
@RequiredArgsConstructor
@@ -125,12 +132,9 @@ public RexNode visitOr(Or node, CalcitePlanContext context) {
125132

126133
@Override
127134
public RexNode visitXor(Xor node, CalcitePlanContext context) {
128-
final RelDataType booleanType =
129-
context.rexBuilder.getTypeFactory().createSqlType(SqlTypeName.BOOLEAN);
130135
final RexNode left = analyze(node.getLeft(), context);
131136
final RexNode right = analyze(node.getRight(), context);
132-
return context.rexBuilder.makeCall(
133-
booleanType, SqlStdOperatorTable.BIT_XOR, List.of(left, right));
137+
return context.relBuilder.notEquals(left, right);
134138
}
135139

136140
@Override
@@ -141,12 +145,28 @@ public RexNode visitNot(Not node, CalcitePlanContext context) {
141145

142146
@Override
143147
public RexNode visitCompare(Compare node, CalcitePlanContext context) {
144-
final RelDataType booleanType =
145-
context.rexBuilder.getTypeFactory().createSqlType(SqlTypeName.BOOLEAN);
148+
SqlOperator op = BuiltinFunctionUtils.translate(node.getOperator());
146149
final RexNode left = analyze(node.getLeft(), context);
147150
final RexNode right = analyze(node.getRight(), context);
148-
return context.rexBuilder.makeCall(
149-
booleanType, BuiltinFunctionUtils.translate(node.getOperator()), List.of(left, right));
151+
return context.relBuilder.call(op, left, right);
152+
}
153+
154+
@Override
155+
public RexNode visitBetween(Between node, CalcitePlanContext context) {
156+
RexNode value = analyze(node.getValue(), context);
157+
RexNode lowerBound = analyze(node.getLowerBound(), context);
158+
RexNode upperBound = analyze(node.getUpperBound(), context);
159+
RelDataType commonType = context.rexBuilder.commonType(value, lowerBound, upperBound);
160+
if (commonType != null) {
161+
lowerBound = context.rexBuilder.makeCast(commonType, lowerBound);
162+
upperBound = context.rexBuilder.makeCast(commonType, upperBound);
163+
} else {
164+
throw new SemanticCheckException(
165+
StringUtils.format(
166+
"BETWEEN expression types are incompatible: [%s, %s, %s]",
167+
value.getType(), lowerBound.getType(), upperBound.getType()));
168+
}
169+
return context.relBuilder.between(value, lowerBound, upperBound);
150170
}
151171

152172
@Override
@@ -337,4 +357,22 @@ private RelNode resolveSubqueryPlan(UnresolvedPlan subquery, CalcitePlanContext
337357
}
338358
return subqueryRel;
339359
}
360+
361+
/*
362+
* Unsupported Expressions of PPL with Calcite for OpenSearch 3.0.0-beta
363+
*/
364+
@Override
365+
public RexNode visitCast(Cast node, CalcitePlanContext context) {
366+
throw new CalciteUnsupportedException("CastWhen function is unsupported in Calcite");
367+
}
368+
369+
@Override
370+
public RexNode visitWhen(When node, CalcitePlanContext context) {
371+
throw new CalciteUnsupportedException("CastWhen function is unsupported in Calcite");
372+
}
373+
374+
@Override
375+
public RexNode visitRelevanceFieldList(RelevanceFieldList node, CalcitePlanContext context) {
376+
throw new CalciteUnsupportedException("Relevance fields expression is unsupported in Calcite");
377+
}
340378
}

‎core/src/main/java/org/opensearch/sql/calcite/ExtendedRexBuilder.java

+6
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55

66
package org.opensearch.sql.calcite;
77

8+
import java.util.Arrays;
89
import java.util.List;
910
import org.apache.calcite.avatica.util.TimeUnit;
1011
import org.apache.calcite.rel.type.RelDataType;
@@ -35,6 +36,11 @@ public RexNode and(RexNode left, RexNode right) {
3536
return this.makeCall(booleanType, SqlStdOperatorTable.AND, List.of(left, right));
3637
}
3738

39+
public RelDataType commonType(RexNode... nodes) {
40+
return this.getTypeFactory()
41+
.leastRestrictive(Arrays.stream(nodes).map(RexNode::getType).toList());
42+
}
43+
3844
public SqlIntervalQualifier createIntervalUntil(SpanUnit unit) {
3945
TimeUnit timeUnit;
4046
switch (unit) {

‎core/src/main/java/org/opensearch/sql/calcite/utils/BuiltinFunctionUtils.java

+4-3
Original file line numberDiff line numberDiff line change
@@ -34,12 +34,11 @@ static SqlOperator translate(String op) {
3434
case "NOT":
3535
return SqlStdOperatorTable.NOT;
3636
case "XOR":
37-
return SqlStdOperatorTable.BIT_XOR;
37+
case "!=":
38+
return SqlStdOperatorTable.NOT_EQUALS;
3839
case "=":
3940
return SqlStdOperatorTable.EQUALS;
4041
case "<>":
41-
case "!=":
42-
return SqlStdOperatorTable.NOT_EQUALS;
4342
case ">":
4443
return SqlStdOperatorTable.GREATER_THAN;
4544
case ">=":
@@ -48,6 +47,8 @@ static SqlOperator translate(String op) {
4847
return SqlStdOperatorTable.LESS_THAN;
4948
case "<=":
5049
return SqlStdOperatorTable.LESS_THAN_OR_EQUAL;
50+
case "REGEXP":
51+
return SqlLibraryOperators.REGEXP;
5152
case "+":
5253
return SqlStdOperatorTable.PLUS;
5354
case "-":

‎core/src/main/java/org/opensearch/sql/exception/CalciteUnsupportedException.java

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package org.opensearch.sql.exception;
7+
8+
public class CalciteUnsupportedException extends QueryEngineException {
9+
10+
public CalciteUnsupportedException(String message) {
11+
super(message);
12+
}
13+
}

‎core/src/main/java/org/opensearch/sql/executor/QueryService.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ public void execute(
9292
fallbackAllowed = settings.getSettingValue(Settings.Key.CALCITE_FALLBACK_ALLOWED);
9393
}
9494
if (!fallbackAllowed) {
95-
throw e;
95+
listener.onFailure(e);
9696
}
9797
LOG.warn("Fallback to V2 query engine since got exception", e);
9898
executePlan(analyze(plan), PlanContext.emptyPlanContext(), listener);

‎integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteOperatorIT.java

+26
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package org.opensearch.sql.calcite.remote;
7+
8+
import java.io.IOException;
9+
import org.junit.Ignore;
10+
import org.opensearch.sql.ppl.OperatorIT;
11+
12+
public class CalciteOperatorIT extends OperatorIT {
13+
14+
@Override
15+
public void init() throws IOException {
16+
enableCalcite();
17+
disallowCalciteFallback();
18+
super.init();
19+
}
20+
21+
@Ignore("https://github.com/opensearch-project/sql/issues/3398")
22+
@Override
23+
public void testModuleOperator() throws IOException {
24+
super.testModuleOperator();
25+
}
26+
}

0 commit comments

Comments
 (0)