QL: case sensitive support in EQL (elastic#56404)

* * StartsWith is case sensitive aware
* Added case sensitivity to EQL configuration
* case_sensitive parameter can be specified when running queries (default
is case insensitive)
* Added STARTS_WITH function to SQL as well

* Add case sensitive aware queryfolder tests

* Address reviews

* Address review #2

docs/reference/sql/functions/string.asciidoc

@@ -422,6 +422,36 @@ SPACE(count) <1>
 include-tagged::{sql-specs}/docs/docs.csv-spec[stringSpace]
 --------------------------------------------------
 
+[[sql-functions-string-startswith]]
+==== `STARTS_WITH`
+
+.Synopsis:
+[source, sql]
+--------------------------------------------------
+STARTS_WITH(
+ source, <1>
+ pattern) <2>
+--------------------------------------------------
+*Input*:
+
+<1> string expression
+<2> string expression
+
+*Output*: boolean value
+
+*Description*: Returns `true` if the source expression starts with the specified pattern, `false` otherwise. The matching is case sensitive.
+If either parameters is `null`, the function returns `null`.
+
+[source, sql]
+--------------------------------------------------
+include-tagged::{sql-specs}/docs/docs.csv-spec[stringStartsWithTrue]
+--------------------------------------------------
+
+[source, sql]
+--------------------------------------------------
+include-tagged::{sql-specs}/docs/docs.csv-spec[stringStartsWithFalse]
+--------------------------------------------------
+
 [[sql-functions-string-substring]]
 ==== `SUBSTRING`
 

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequest.java

@@ -48,6 +48,7 @@ public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Re
  private int fetchSize = FETCH_SIZE;
  private SearchAfterBuilder searchAfterBuilder;
  private String query;
+ private boolean isCaseSensitive = false;
 
  static final String KEY_FILTER = "filter";
  static final String KEY_TIMESTAMP_FIELD = "timestamp_field";
@@ -56,6 +57,7 @@ public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Re
  static final String KEY_SIZE = "size";
  static final String KEY_SEARCH_AFTER = "search_after";
  static final String KEY_QUERY = "query";
+ static final String KEY_CASE_SENSITIVE = "case_sensitive";
 
  static final ParseField FILTER = new ParseField(KEY_FILTER);
  static final ParseField TIMESTAMP_FIELD = new ParseField(KEY_TIMESTAMP_FIELD);
@@ -64,6 +66,7 @@ public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Re
  static final ParseField SIZE = new ParseField(KEY_SIZE);
  static final ParseField SEARCH_AFTER = new ParseField(KEY_SEARCH_AFTER);
  static final ParseField QUERY = new ParseField(KEY_QUERY);
+ static final ParseField CASE_SENSITIVE = new ParseField(KEY_CASE_SENSITIVE);
 
  private static final ObjectParser<EqlSearchRequest, Void> PARSER = objectParser(EqlSearchRequest::new);
 
@@ -82,6 +85,7 @@ public EqlSearchRequest(StreamInput in) throws IOException {
  fetchSize = in.readVInt();
  searchAfterBuilder = in.readOptionalWriteable(SearchAfterBuilder::new);
  query = in.readString();
+ isCaseSensitive = in.readBoolean();
  }
 
  @Override
@@ -143,6 +147,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
  }
 
  builder.field(KEY_QUERY, query);
+ builder.field(KEY_CASE_SENSITIVE, isCaseSensitive);
 
  return builder;
  }
@@ -162,6 +167,7 @@ protected static <R extends EqlSearchRequest> ObjectParser<R, Void> objectParser
  parser.declareField(EqlSearchRequest::setSearchAfter, SearchAfterBuilder::fromXContent, SEARCH_AFTER,
  ObjectParser.ValueType.OBJECT_ARRAY);
  parser.declareString(EqlSearchRequest::query, QUERY);
+ parser.declareBoolean(EqlSearchRequest::isCaseSensitive, CASE_SENSITIVE);
  return parser;
  }
 
@@ -230,6 +236,13 @@ public EqlSearchRequest query(String query) {
  return this;
  }
 
+ public boolean isCaseSensitive() { return this.isCaseSensitive; }
+
+ public EqlSearchRequest isCaseSensitive(boolean isCaseSensitive) {
+ this.isCaseSensitive = isCaseSensitive;
+ return this;
+ }
+
  @Override
  public void writeTo(StreamOutput out) throws IOException {
  super.writeTo(out);
@@ -242,6 +255,7 @@ public void writeTo(StreamOutput out) throws IOException {
  out.writeVInt(fetchSize);
  out.writeOptionalWriteable(searchAfterBuilder);
  out.writeString(query);
+ out.writeBoolean(isCaseSensitive);
  }
 
  @Override
@@ -261,7 +275,8 @@ public boolean equals(Object o) {
  Objects.equals(eventCategoryField, that.eventCategoryField) &&
  Objects.equals(implicitJoinKeyField, that.implicitJoinKeyField) &&
  Objects.equals(searchAfterBuilder, that.searchAfterBuilder) &&
- Objects.equals(query, that.query);
+ Objects.equals(query, that.query) &&
+ Objects.equals(isCaseSensitive, that.isCaseSensitive);
  }
 
  @Override
@@ -274,7 +289,8 @@ public int hashCode() {
  timestampField, eventCategoryField,
  implicitJoinKeyField,
  searchAfterBuilder,
- query);
+ query,
+ isCaseSensitive);
  }
 
  @Override

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequestBuilder.java

@@ -55,4 +55,8 @@ public EqlSearchRequestBuilder query(String query) {
  return this;
  }
 
+ public EqlSearchRequestBuilder query(boolean isCaseSensitive) {
+ request.isCaseSensitive(isCaseSensitive);
+ return this;
+ }
 }

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/analysis/Analyzer.java

@@ -16,6 +16,7 @@
 import org.elasticsearch.xpack.ql.expression.function.UnresolvedFunction;
 import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan;
 import org.elasticsearch.xpack.ql.rule.RuleExecutor;
+import org.elasticsearch.xpack.ql.session.Configuration;
 
 import java.util.ArrayList;
 import java.util.Collection;
@@ -26,10 +27,12 @@
 
 public class Analyzer extends RuleExecutor<LogicalPlan> {
 
+ private final Configuration configuration;
  private final FunctionRegistry functionRegistry;
  private final Verifier verifier;
 
- public Analyzer(FunctionRegistry functionRegistry, Verifier verifier) {
+ public Analyzer(Configuration configuration, FunctionRegistry functionRegistry, Verifier verifier) {
+ this.configuration = configuration;
  this.functionRegistry = functionRegistry;
  this.verifier = verifier;
  }
@@ -113,7 +116,7 @@ protected LogicalPlan rule(LogicalPlan plan) {
  return uf.missing(functionName, functionRegistry.listFunctions());
  }
  FunctionDefinition def = functionRegistry.resolveFunction(functionName);
- Function f = uf.buildResolved(null, def);
+ Function f = uf.buildResolved(configuration, def);
  return f;
  }
  return e;

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/PlanExecutor.java

@@ -9,14 +9,13 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.xpack.eql.analysis.Analyzer;
 import org.elasticsearch.xpack.eql.analysis.PreAnalyzer;
 import org.elasticsearch.xpack.eql.analysis.Verifier;
 import org.elasticsearch.xpack.eql.expression.function.EqlFunctionRegistry;
 import org.elasticsearch.xpack.eql.optimizer.Optimizer;
 import org.elasticsearch.xpack.eql.parser.ParserParams;
 import org.elasticsearch.xpack.eql.planner.Planner;
-import org.elasticsearch.xpack.eql.session.Configuration;
+import org.elasticsearch.xpack.eql.session.EqlConfiguration;
 import org.elasticsearch.xpack.eql.session.EqlSession;
 import org.elasticsearch.xpack.eql.session.Results;
 import org.elasticsearch.xpack.eql.stats.Metrics;
@@ -33,7 +32,7 @@ public class PlanExecutor {
  private final FunctionRegistry functionRegistry;
 
  private final PreAnalyzer preAnalyzer;
- private final Analyzer analyzer;
+ private final Verifier verifier;
  private final Optimizer optimizer;
  private final Planner planner;
 
@@ -50,16 +49,16 @@ public PlanExecutor(Client client, IndexResolver indexResolver, NamedWriteableRe
  this.metrics = new Metrics();
 
  this.preAnalyzer = new PreAnalyzer();
- this.analyzer = new Analyzer(functionRegistry, new Verifier());
+ this.verifier = new Verifier();
  this.optimizer = new Optimizer();
  this.planner = new Planner();
  }
 
- private EqlSession newSession(Configuration cfg) {
- return new EqlSession(client, cfg, indexResolver, preAnalyzer, analyzer, optimizer, planner, this);
+ private EqlSession newSession(EqlConfiguration cfg) {
+ return new EqlSession(client, cfg, indexResolver, preAnalyzer, functionRegistry, verifier, optimizer, planner, this);
  }
 
- public void eql(Configuration cfg, String eql, ParserParams parserParams, ActionListener<Results> listener) {
+ public void eql(EqlConfiguration cfg, String eql, ParserParams parserParams, ActionListener<Results> listener) {
  newSession(cfg).eql(eql, parserParams, wrap(listener::onResponse, listener::onFailure));
  }
 

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/Querier.java

@@ -19,7 +19,7 @@
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.elasticsearch.tasks.TaskCancelledException;
 import org.elasticsearch.xpack.eql.querydsl.container.QueryContainer;
-import org.elasticsearch.xpack.eql.session.Configuration;
+import org.elasticsearch.xpack.eql.session.EqlConfiguration;
 import org.elasticsearch.xpack.eql.session.EqlSession;
 import org.elasticsearch.xpack.eql.session.Results;
 import org.elasticsearch.xpack.ql.expression.Attribute;
@@ -33,7 +33,7 @@ public class Querier {
 
  private static final Logger log = LogManager.getLogger(Querier.class);
 
- private final Configuration cfg;
+ private final EqlConfiguration cfg;
  private final Client client;
  private final TimeValue keepAlive;
  private final QueryBuilder filter;

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/SearchAfterListener.java

@@ -20,7 +20,7 @@
 import org.elasticsearch.xpack.eql.querydsl.container.ComputedRef;
 import org.elasticsearch.xpack.eql.querydsl.container.QueryContainer;
 import org.elasticsearch.xpack.eql.querydsl.container.SearchHitFieldRef;
-import org.elasticsearch.xpack.eql.session.Configuration;
+import org.elasticsearch.xpack.eql.session.EqlConfiguration;
 import org.elasticsearch.xpack.eql.session.Results;
 import org.elasticsearch.xpack.ql.execution.search.FieldExtraction;
 import org.elasticsearch.xpack.ql.execution.search.extractor.ComputingExtractor;
@@ -43,12 +43,12 @@ class SearchAfterListener implements ActionListener<SearchResponse> {
  private final ActionListener<Results> listener;
 
  private final Client client;
- private final Configuration cfg;
+ private final EqlConfiguration cfg;
  private final List<Attribute> output;
  private final QueryContainer container;
  private final SearchRequest request;
 
- SearchAfterListener(ActionListener<Results> listener, Client client, Configuration cfg, List<Attribute> output,
+ SearchAfterListener(ActionListener<Results> listener, Client client, EqlConfiguration cfg, List<Attribute> output,
  QueryContainer container, SearchRequest request) {
 
  this.listener = listener;

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/EqlFunctionRegistry.java

@@ -6,8 +6,8 @@
 
 package org.elasticsearch.xpack.eql.expression.function;
 
-import org.elasticsearch.xpack.eql.expression.function.scalar.string.CIDRMatch;
 import org.elasticsearch.xpack.eql.expression.function.scalar.string.Between;
+import org.elasticsearch.xpack.eql.expression.function.scalar.string.CIDRMatch;
 import org.elasticsearch.xpack.eql.expression.function.scalar.string.Concat;
 import org.elasticsearch.xpack.eql.expression.function.scalar.string.EndsWith;
 import org.elasticsearch.xpack.eql.expression.function.scalar.string.IndexOf;
@@ -50,15 +50,15 @@ private static FunctionDefinition[][] functions() {
  def(ToString.class, ToString::new, "string"),
  def(StringContains.class, StringContains::new, "stringcontains"),
  def(Substring.class, Substring::new, "substring"),
- def(Wildcard.class, Wildcard::new, "wildcard"),
+ def(Wildcard.class, Wildcard::new, "wildcard")
  },
  // Arithmetic
  new FunctionDefinition[] {
-  def(Add.class, Add::new, "add"),
-  def(Div.class, Div::new, "divide"),
-  def(Mod.class, Mod::new, "modulo"),
-  def(Mul.class, Mul::new, "multiply"),
-  def(Sub.class, Sub::new, "subtract"),
+ def(Add.class, Add::new, "add"),
+ def(Div.class, Div::new, "divide"),
+ def(Mod.class, Mod::new, "modulo"),
+ def(Mul.class, Mul::new, "multiply"),
+ def(Sub.class, Sub::new, "subtract")
  }
  };
  }