Support In clause in SQL and PPL (opensearch-project#420)

Signed-off-by: penghuo <[email protected]>

Author: penghuo

core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java

@@ -26,6 +26,7 @@
 import org.opensearch.sql.ast.expression.EqualTo;
 import org.opensearch.sql.ast.expression.Field;
 import org.opensearch.sql.ast.expression.Function;
+import org.opensearch.sql.ast.expression.In;
 import org.opensearch.sql.ast.expression.Interval;
 import org.opensearch.sql.ast.expression.Literal;
 import org.opensearch.sql.ast.expression.Not;
@@ -177,6 +178,24 @@ public Expression visitWindowFunction(WindowFunction node, AnalysisContext conte
     return expr;
   }
 
+  @Override
+  public Expression visitIn(In node, AnalysisContext context) {
+    return visitIn(node.getField(), node.getValueList(), context);
+  }
+
+  private Expression visitIn(
+      UnresolvedExpression field, List<UnresolvedExpression> valueList, AnalysisContext context) {
+    if (valueList.size() == 1) {
+      return visitCompare(new Compare("=", field, valueList.get(0)), context);
+    } else if (valueList.size() > 1) {
+      return dsl.or(
+          visitCompare(new Compare("=", field, valueList.get(0)), context),
+          visitIn(field, valueList.subList(1, valueList.size()), context));
+    } else {
+      throw new SemanticCheckException("Values in In clause should not be empty");
+    }
+  }
+
   @Override
   public Expression visitCompare(Compare node, AnalysisContext context) {
     FunctionName functionName = FunctionName.of(node.getOperator());

core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java

@@ -279,6 +279,11 @@ public static UnresolvedExpression in(
     return new In(field, Arrays.asList(valueList));
   }
 
+  public static UnresolvedExpression in(
+      UnresolvedExpression field, List<UnresolvedExpression> valueList) {
+    return new In(field, valueList);
+  }
+
   public static UnresolvedExpression compare(
       String operator, UnresolvedExpression left, UnresolvedExpression right) {
     return new Compare(operator, left, right);

core/src/test/java/org/opensearch/sql/analysis/ExpressionAnalyzerTest.java

@@ -18,25 +18,23 @@
 import static org.opensearch.sql.data.model.ExprValueUtils.integerValue;
 import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN;
 import static org.opensearch.sql.data.type.ExprCoreType.INTEGER;
-import static org.opensearch.sql.data.type.ExprCoreType.STRING;
 import static org.opensearch.sql.data.type.ExprCoreType.STRUCT;
 
+import java.util.Collections;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.opensearch.sql.analysis.symbol.Namespace;
 import org.opensearch.sql.analysis.symbol.Symbol;
 import org.opensearch.sql.ast.dsl.AstDSL;
 import org.opensearch.sql.ast.expression.AllFields;
 import org.opensearch.sql.ast.expression.DataType;
-import org.opensearch.sql.ast.expression.Literal;
 import org.opensearch.sql.ast.expression.SpanUnit;
 import org.opensearch.sql.ast.expression.UnresolvedExpression;
 import org.opensearch.sql.common.antlr.SyntaxCheckException;
 import org.opensearch.sql.data.model.ExprValueUtils;
 import org.opensearch.sql.exception.SemanticCheckException;
 import org.opensearch.sql.expression.DSL;
 import org.opensearch.sql.expression.Expression;
-import org.opensearch.sql.expression.NamedExpression;
 import org.opensearch.sql.expression.config.ExpressionConfig;
 import org.opensearch.sql.expression.window.aggregation.AggregateWindowFunction;
 import org.springframework.context.annotation.Configuration;
@@ -319,6 +317,21 @@ void visit_span() {
     );
   }
 
+  @Test
+  void visit_in() {
+    assertAnalyzeEqual(
+        dsl.or(
+            dsl.equal(DSL.ref("integer_value", INTEGER), DSL.literal(1)),
+            dsl.or(
+                dsl.equal(DSL.ref("integer_value", INTEGER), DSL.literal(2)),
+                dsl.equal(DSL.ref("integer_value", INTEGER), DSL.literal(3)))),
+        AstDSL.in(field("integer_value"), intLiteral(1), intLiteral(2), intLiteral(3)));
+
+    assertThrows(
+        SemanticCheckException.class,
+        () -> analyze(AstDSL.in(field("integer_value"), Collections.emptyList())));
+  }
+
   protected Expression analyze(UnresolvedExpression unresolvedExpression) {
     return expressionAnalyzer.analyze(unresolvedExpression, analysisContext);
   }

docs/category.json

@@ -21,7 +21,8 @@
     "user/ppl/functions/datetime.rst",
     "user/ppl/functions/string.rst",
     "user/ppl/functions/condition.rst",
-    "user/ppl/functions/relevance.rst"
+    "user/ppl/functions/relevance.rst",
+    "user/ppl/functions/expressions.rst"
   ],
   "sql_cli": [
     "user/dql/expressions.rst",
@@ -36,4 +37,4 @@
     "user/dql/aggregations.rst",
     "user/dql/complex.rst"
   ]
-}
+}

docs/user/dql/expressions.rst

@@ -128,7 +128,10 @@ Operators
 +----------------+----------------------------------------+
 | REGEXP         | String matches regular expression test |
 +----------------+----------------------------------------+
-
+| IN             | IN value list test                     |
++----------------+----------------------------------------+
+| NOT IN         | NOT IN value list test                 |
++----------------+----------------------------------------+
 
 Basic Comparison Operator
 -------------------------
@@ -183,6 +186,19 @@ expr REGEXP pattern. The expr is string value, pattern is supports regular expre
     | 1                      | 0                |
     +------------------------+------------------+
 
+IN value list test
+------------------
+
+Here is an example for IN value test::
+
+    os> SELECT 1 in (1, 2), 3 not in (1, 2);
+    fetched rows / total rows = 1/1
+    +---------------+-------------------+
+    | 1 in (1, 2)   | 3 not in (1, 2)   |
+    |---------------+-------------------|
+    | True          | True              |
+    +---------------+-------------------+
+
 Function Call
 =============
 

docs/user/ppl/functions/expressions.rst

@@ -0,0 +1,158 @@
+===========
+Expressions
+===========
+
+.. rubric:: Table of contents
+
+.. contents::
+   :local:
+   :depth: 3
+
+
+Introduction
+============
+
+Expressions, particularly value expressions, are those which return a scalar value. Expressions have different types and forms. For example, there are literal values as atom expression and arithmetic, predicate and function expression built on top of them. And also expressions can be used in different clauses, such as using arithmetic expression in ``Filter``, ``Stats`` command.
+
+Arithmetic Operators
+====================
+
+Description
+-----------
+
+Operators
+`````````
+
+Arithmetic expression is an expression formed by numeric literals and binary arithmetic operators as follows:
+
+1. ``+``: Add.
+2. ``-``: Subtract.
+3. ``*``: Multiply.
+4. ``/``: Divide. For integers, the result is an integer with fractional part discarded.
+5. ``%``: Modulo. This can be used with integers only with remainder of the division as result.
+
+Precedence
+``````````
+
+Parentheses can be used to control the precedence of arithmetic operators. Otherwise, operators of higher precedence is performed first.
+
+Type Conversion
+```````````````
+
+Implicit type conversion is performed when looking up operator signature. For example, an integer ``+`` a real number matches signature ``+(double,double)`` which results in a real number. This rule also applies to function call discussed below.
+
+Examples
+--------
+
+Here is an example for different type of arithmetic expressions::
+
+    os> source=accounts | where age > (25 + 5) | fields age ;
+    fetched rows / total rows = 3/3
+    +-------+
+    | age   |
+    |-------|
+    | 32    |
+    | 36    |
+    | 33    |
+    +-------+
+
+Predicate Operators
+===================
+
+Description
+-----------
+
+Predicate operator is an expression that evaluated to be ture. The MISSING and NULL value comparison has following the rule. MISSING value only equal to MISSING value and less than all the other values. NULL value equals to NULL value, large than MISSING value, but less than all the other values.
+
+Operators
+`````````
+
++----------------+----------------------------------------+
+| name           | description                            |
++----------------+----------------------------------------+
+| >              | Greater than operator                  |
++----------------+----------------------------------------+
+| >=             | Greater than or equal operator         |
++----------------+----------------------------------------+
+| <              | Less than operator                     |
++----------------+----------------------------------------+
+| !=             | Not equal operator                     |
++----------------+----------------------------------------+
+| <=             | Less than or equal operator            |
++----------------+----------------------------------------+
+| =              | Equal operator                         |
++----------------+----------------------------------------+
+| LIKE           | Simple Pattern matching                |
++----------------+----------------------------------------+
+| IN             | NULL value test                        |
++----------------+----------------------------------------+
+| AND            | AND operator                           |
++----------------+----------------------------------------+
+| OR             | OR operator                            |
++----------------+----------------------------------------+
+| XOR            | XOR operator                           |
++----------------+----------------------------------------+
+| NOT            | NOT NULL value test                    |
++----------------+----------------------------------------+
+
+Examples
+--------
+
+Basic Predicate Operator
+````````````````````````
+
+Here is an example for comparison operators::
+
+    os> source=accounts | where age > 33 | fields age ;
+    fetched rows / total rows = 1/1
+    +-------+
+    | age   |
+    |-------|
+    | 36    |
+    +-------+
+
+
+IN
+``
+
+IN operator test field in value lists::
+
+    os> source=accounts | where age in (32, 33) | fields age ;
+    fetched rows / total rows = 2/2
+    +-------+
+    | age   |
+    |-------|
+    | 32    |
+    | 33    |
+    +-------+
+
+
+OR
+``
+
+OR operator ::
+
+    os> source=accounts | where age = 32 OR age = 33 | fields age ;
+    fetched rows / total rows = 2/2
+    +-------+
+    | age   |
+    |-------|
+    | 32    |
+    | 33    |
+    +-------+
+
+
+NOT
+```
+
+NOT operator ::
+
+    os> source=accounts | where not age in (32, 33) | fields age ;
+    fetched rows / total rows = 2/2
+    +-------+
+    | age   |
+    |-------|
+    | 36    |
+    | 28    |
+    +-------+
+

docs/user/ppl/index.rst

@@ -60,6 +60,8 @@ The query start with search command and then flowing a set of command delimited
 
 * **Functions**
 
+  - `Expressions <functions/expressions.rst>`_
+
   - `Math Functions <functions/math.rst>`_
 
   - `Date and Time Functions <functions/datetime.rst>`_
@@ -82,4 +84,4 @@ The query start with search command and then flowing a set of command delimited
 
 * **Limitations**
 
-  - `Limitations <limitations/limitations.rst>`_
+  - `Limitations <limitations/limitations.rst>`_

integ-test/src/test/java/org/opensearch/sql/legacy/CsvFormatResponseIT.java

@@ -384,10 +384,10 @@ public void aggAfterTwoTermsGroupBy() throws Exception {
     List<String> lines = csvResult.getLines();
     Assert.assertEquals(4, lines.size());
     assertThat(lines, containsInAnyOrder(
-        equalTo("31.0"),
-        equalTo("28.0"),
-        equalTo("21.0"),
-        equalTo("24.0")));
+        equalTo("31"),
+        equalTo("28"),
+        equalTo("21"),
+        equalTo("24")));
   }
 
   @Test
@@ -398,15 +398,15 @@ public void multipleAggAfterTwoTermsGroupBy() throws Exception {
     CSVResult csvResult = executeCsvRequest(query, false);
     List<String> headers = csvResult.getHeaders();
     Assert.assertEquals(2, headers.size());
-    assertThat(headers, contains(equalTo("COUNT(*)"), equalTo("SUM(balance)")));
+    assertThat(headers, contains(equalTo("COUNT(*)"), equalTo("sum(balance)")));
 
     List<String> lines = csvResult.getLines();
     Assert.assertEquals(4, lines.size());
     assertThat(lines, containsInAnyOrder(
-        equalTo("31.0,647425.0"),
-        equalTo("28.0,678337.0"),
-        equalTo("21.0,505660.0"),
-        equalTo("24.0,472771.0")));
+        equalTo("31,647425"),
+        equalTo("28,678337"),
+        equalTo("21,505660"),
+        equalTo("24,472771")));
   }
 
   @Test

ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java

@@ -48,7 +48,6 @@
 import java.util.stream.Collectors;
 import org.antlr.v4.runtime.ParserRuleContext;
 import org.antlr.v4.runtime.RuleContext;
-import org.antlr.v4.runtime.Token;
 import org.opensearch.sql.ast.expression.AggregateFunction;
 import org.opensearch.sql.ast.expression.Alias;
 import org.opensearch.sql.ast.expression.AllFields;
@@ -72,7 +71,6 @@
 import org.opensearch.sql.ast.expression.UnresolvedExpression;
 import org.opensearch.sql.ast.expression.Xor;
 import org.opensearch.sql.common.utils.StringUtils;
-import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser;
 import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParserBaseVisitor;
 import org.opensearch.sql.ppl.utils.ArgumentFactory;
 

ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java

@@ -10,7 +10,6 @@
 import static org.opensearch.sql.ast.dsl.AstDSL.agg;
 import static org.opensearch.sql.ast.dsl.AstDSL.aggregate;
 import static org.opensearch.sql.ast.dsl.AstDSL.alias;
-import static org.opensearch.sql.ast.dsl.AstDSL.allFields;
 import static org.opensearch.sql.ast.dsl.AstDSL.and;
 import static org.opensearch.sql.ast.dsl.AstDSL.argument;
 import static org.opensearch.sql.ast.dsl.AstDSL.booleanLiteral;