QL: case sensitive support in EQL (elastic#56404)

* * StartsWith is case sensitive aware
* Added case sensitivity to EQL configuration
* case_sensitive parameter can be specified when running queries (default
is case insensitive)
* Added STARTS_WITH function to SQL as well

* Add case sensitive aware queryfolder tests

* Address reviews

* Address review #2

Author: astefan

docs/reference/sql/functions/string.asciidoc

@@ -422,6 +422,36 @@ SPACE(count) <1>
 include-tagged::{sql-specs}/docs/docs.csv-spec[stringSpace]
 --------------------------------------------------
 
+[[sql-functions-string-startswith]]
+==== `STARTS_WITH`
+
+.Synopsis:
+[source, sql]
+--------------------------------------------------
+STARTS_WITH(
+    source,   <1>
+    pattern)  <2>
+--------------------------------------------------
+*Input*:
+
+<1> string expression
+<2> string expression
+
+*Output*: boolean value
+
+*Description*: Returns `true` if the source expression starts with the specified pattern, `false` otherwise. The matching is case sensitive.
+If either parameters is `null`, the function returns `null`.
+
+[source, sql]
+--------------------------------------------------
+include-tagged::{sql-specs}/docs/docs.csv-spec[stringStartsWithTrue]
+--------------------------------------------------
+
+[source, sql]
+--------------------------------------------------
+include-tagged::{sql-specs}/docs/docs.csv-spec[stringStartsWithFalse]
+--------------------------------------------------
+
 [[sql-functions-string-substring]]
 ==== `SUBSTRING`
 

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequest.java

@@ -48,6 +48,7 @@ public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Re
     private int fetchSize = FETCH_SIZE;
     private SearchAfterBuilder searchAfterBuilder;
     private String query;
+    private boolean isCaseSensitive = false;
 
     static final String KEY_FILTER = "filter";
     static final String KEY_TIMESTAMP_FIELD = "timestamp_field";
@@ -56,6 +57,7 @@ public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Re
     static final String KEY_SIZE = "size";
     static final String KEY_SEARCH_AFTER = "search_after";
     static final String KEY_QUERY = "query";
+    static final String KEY_CASE_SENSITIVE = "case_sensitive";
 
     static final ParseField FILTER = new ParseField(KEY_FILTER);
     static final ParseField TIMESTAMP_FIELD = new ParseField(KEY_TIMESTAMP_FIELD);
@@ -64,6 +66,7 @@ public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Re
     static final ParseField SIZE = new ParseField(KEY_SIZE);
     static final ParseField SEARCH_AFTER = new ParseField(KEY_SEARCH_AFTER);
     static final ParseField QUERY = new ParseField(KEY_QUERY);
+    static final ParseField CASE_SENSITIVE = new ParseField(KEY_CASE_SENSITIVE);
 
     private static final ObjectParser<EqlSearchRequest, Void> PARSER = objectParser(EqlSearchRequest::new);
 
@@ -82,6 +85,7 @@ public EqlSearchRequest(StreamInput in) throws IOException {
         fetchSize = in.readVInt();
         searchAfterBuilder = in.readOptionalWriteable(SearchAfterBuilder::new);
         query = in.readString();
+        isCaseSensitive = in.readBoolean();
     }
 
     @Override
@@ -143,6 +147,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         }
 
         builder.field(KEY_QUERY, query);
+        builder.field(KEY_CASE_SENSITIVE, isCaseSensitive);
 
         return builder;
     }
@@ -162,6 +167,7 @@ protected static <R extends EqlSearchRequest> ObjectParser<R, Void> objectParser
         parser.declareField(EqlSearchRequest::setSearchAfter, SearchAfterBuilder::fromXContent, SEARCH_AFTER,
             ObjectParser.ValueType.OBJECT_ARRAY);
         parser.declareString(EqlSearchRequest::query, QUERY);
+        parser.declareBoolean(EqlSearchRequest::isCaseSensitive, CASE_SENSITIVE);
         return parser;
     }
 
@@ -230,6 +236,13 @@ public EqlSearchRequest query(String query) {
         return this;
     }
 
+    public boolean isCaseSensitive() { return this.isCaseSensitive; }
+
+    public EqlSearchRequest isCaseSensitive(boolean isCaseSensitive) {
+        this.isCaseSensitive = isCaseSensitive;
+        return this;
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
@@ -242,6 +255,7 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeVInt(fetchSize);
         out.writeOptionalWriteable(searchAfterBuilder);
         out.writeString(query);
+        out.writeBoolean(isCaseSensitive);
     }
 
     @Override
@@ -261,7 +275,8 @@ public boolean equals(Object o) {
                 Objects.equals(eventCategoryField, that.eventCategoryField) &&
                 Objects.equals(implicitJoinKeyField, that.implicitJoinKeyField) &&
                 Objects.equals(searchAfterBuilder, that.searchAfterBuilder) &&
-                Objects.equals(query, that.query);
+                Objects.equals(query, that.query) &&
+                Objects.equals(isCaseSensitive, that.isCaseSensitive);
     }
 
     @Override
@@ -274,7 +289,8 @@ public int hashCode() {
             timestampField, eventCategoryField,
             implicitJoinKeyField,
             searchAfterBuilder,
-            query);
+            query,
+            isCaseSensitive);
     }
 
     @Override

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequestBuilder.java

@@ -55,4 +55,8 @@ public EqlSearchRequestBuilder query(String query) {
         return this;
     }
 
+    public EqlSearchRequestBuilder query(boolean isCaseSensitive) {
+        request.isCaseSensitive(isCaseSensitive);
+        return this;
+    }
 }

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/Analyzer.java

@@ -16,6 +16,7 @@
 import org.elasticsearch.xpack.ql.expression.function.UnresolvedFunction;
 import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan;
 import org.elasticsearch.xpack.ql.rule.RuleExecutor;
+import org.elasticsearch.xpack.ql.session.Configuration;
 
 import java.util.ArrayList;
 import java.util.Collection;
@@ -26,10 +27,12 @@
 
 public class Analyzer extends RuleExecutor<LogicalPlan> {
 
+    private final Configuration configuration;
     private final FunctionRegistry functionRegistry;
     private final Verifier verifier;
 
-    public Analyzer(FunctionRegistry functionRegistry, Verifier verifier) {
+    public Analyzer(Configuration configuration, FunctionRegistry functionRegistry, Verifier verifier) {
+        this.configuration = configuration;
         this.functionRegistry = functionRegistry;
         this.verifier = verifier;
     }
@@ -113,7 +116,7 @@ protected LogicalPlan rule(LogicalPlan plan) {
                         return uf.missing(functionName, functionRegistry.listFunctions());
                     }
                     FunctionDefinition def = functionRegistry.resolveFunction(functionName);
-                    Function f = uf.buildResolved(null, def);
+                    Function f = uf.buildResolved(configuration, def);
                     return f;
                 }
                 return e;

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/PlanExecutor.java

@@ -9,14 +9,13 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.xpack.eql.analysis.Analyzer;
 import org.elasticsearch.xpack.eql.analysis.PreAnalyzer;
 import org.elasticsearch.xpack.eql.analysis.Verifier;
 import org.elasticsearch.xpack.eql.expression.function.EqlFunctionRegistry;
 import org.elasticsearch.xpack.eql.optimizer.Optimizer;
 import org.elasticsearch.xpack.eql.parser.ParserParams;
 import org.elasticsearch.xpack.eql.planner.Planner;
-import org.elasticsearch.xpack.eql.session.Configuration;
+import org.elasticsearch.xpack.eql.session.EqlConfiguration;
 import org.elasticsearch.xpack.eql.session.EqlSession;
 import org.elasticsearch.xpack.eql.session.Results;
 import org.elasticsearch.xpack.eql.stats.Metrics;
@@ -33,7 +32,7 @@ public class PlanExecutor {
     private final FunctionRegistry functionRegistry;
 
     private final PreAnalyzer preAnalyzer;
-    private final Analyzer analyzer;
+    private final Verifier verifier;
     private final Optimizer optimizer;
     private final Planner planner;
 
@@ -50,16 +49,16 @@ public PlanExecutor(Client client, IndexResolver indexResolver, NamedWriteableRe
         this.metrics = new Metrics();
 
         this.preAnalyzer = new PreAnalyzer();
-        this.analyzer = new Analyzer(functionRegistry, new Verifier());
+        this.verifier = new Verifier();
         this.optimizer = new Optimizer();
         this.planner = new Planner();
     }
 
-    private EqlSession newSession(Configuration cfg) {
-        return new EqlSession(client, cfg, indexResolver, preAnalyzer, analyzer, optimizer, planner, this);
+    private EqlSession newSession(EqlConfiguration cfg) {
+        return new EqlSession(client, cfg, indexResolver, preAnalyzer, functionRegistry, verifier, optimizer, planner, this);
     }
 
-    public void eql(Configuration cfg, String eql, ParserParams parserParams, ActionListener<Results> listener) {
+    public void eql(EqlConfiguration cfg, String eql, ParserParams parserParams, ActionListener<Results> listener) {
         newSession(cfg).eql(eql, parserParams, wrap(listener::onResponse, listener::onFailure));
     }
 

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/Querier.java

@@ -19,7 +19,7 @@
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.tasks.TaskCancelledException;
 import org.elasticsearch.xpack.eql.querydsl.container.QueryContainer;
-import org.elasticsearch.xpack.eql.session.Configuration;
+import org.elasticsearch.xpack.eql.session.EqlConfiguration;
 import org.elasticsearch.xpack.eql.session.EqlSession;
 import org.elasticsearch.xpack.eql.session.Results;
 import org.elasticsearch.xpack.ql.expression.Attribute;
@@ -33,7 +33,7 @@ public class Querier {
 
     private static final Logger log = LogManager.getLogger(Querier.class);
 
-    private final Configuration cfg;
+    private final EqlConfiguration cfg;
     private final Client client;
     private final TimeValue keepAlive;
     private final QueryBuilder filter;

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/SearchAfterListener.java

@@ -20,7 +20,7 @@
 import org.elasticsearch.xpack.eql.querydsl.container.ComputedRef;
 import org.elasticsearch.xpack.eql.querydsl.container.QueryContainer;
 import org.elasticsearch.xpack.eql.querydsl.container.SearchHitFieldRef;
-import org.elasticsearch.xpack.eql.session.Configuration;
+import org.elasticsearch.xpack.eql.session.EqlConfiguration;
 import org.elasticsearch.xpack.eql.session.Results;
 import org.elasticsearch.xpack.ql.execution.search.FieldExtraction;
 import org.elasticsearch.xpack.ql.execution.search.extractor.ComputingExtractor;
@@ -43,12 +43,12 @@ class SearchAfterListener implements ActionListener<SearchResponse> {
     private final ActionListener<Results> listener;
 
     private final Client client;
-    private final Configuration cfg;
+    private final EqlConfiguration cfg;
     private final List<Attribute> output;
     private final QueryContainer container;
     private final SearchRequest request;
 
-    SearchAfterListener(ActionListener<Results> listener, Client client, Configuration cfg, List<Attribute> output,
+    SearchAfterListener(ActionListener<Results> listener, Client client, EqlConfiguration cfg, List<Attribute> output,
                                QueryContainer container, SearchRequest request) {
 
         this.listener = listener;

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java

@@ -6,8 +6,8 @@
 
 package org.elasticsearch.xpack.eql.expression.function;
 
-import org.elasticsearch.xpack.eql.expression.function.scalar.string.CIDRMatch;
 import org.elasticsearch.xpack.eql.expression.function.scalar.string.Between;
+import org.elasticsearch.xpack.eql.expression.function.scalar.string.CIDRMatch;
 import org.elasticsearch.xpack.eql.expression.function.scalar.string.Concat;
 import org.elasticsearch.xpack.eql.expression.function.scalar.string.EndsWith;
 import org.elasticsearch.xpack.eql.expression.function.scalar.string.IndexOf;
@@ -50,15 +50,15 @@ private static FunctionDefinition[][] functions() {
                 def(ToString.class, ToString::new, "string"),
                 def(StringContains.class, StringContains::new, "stringcontains"),
                 def(Substring.class, Substring::new, "substring"),
-                def(Wildcard.class, Wildcard::new, "wildcard"),
+                def(Wildcard.class, Wildcard::new, "wildcard")
             },
         // Arithmetic
             new FunctionDefinition[] {
-                    def(Add.class, Add::new, "add"),
-                    def(Div.class, Div::new, "divide"),
-                    def(Mod.class, Mod::new, "modulo"),
-                    def(Mul.class, Mul::new, "multiply"),
-                    def(Sub.class, Sub::new, "subtract"),
+                def(Add.class, Add::new, "add"),
+                def(Div.class, Div::new, "divide"),
+                def(Mod.class, Mod::new, "modulo"),
+                def(Mul.class, Mul::new, "multiply"),
+                def(Sub.class, Sub::new, "subtract")
             }
         };
     }