EQL: Add optional fields and limit joining keys on non-null values only

x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/BaseEqlSpecTestCase.java

@@ -44,6 +44,11 @@ public abstract class BaseEqlSpecTestCase extends RemoteClusterAwareEqlRestTestC
  private final String query;
  private final String name;
  private final long[] eventIds;
+ /**
+ * Join keys can be of multiple types, but toml is very restrictive and doesn't allow mixed types values in the same array of values
+ * For now, every value will be converted to a String.
+ */
+ private final String[] joinKeys;
 
  @Before
  public void setup() throws Exception {
@@ -78,18 +83,19 @@ protected static List<Object[]> asArray(List<EqlSpec> specs) {
  name = "" + (counter);
  }
 
- results.add(new Object[] { spec.query(), name, spec.expectedEventIds() });
+ results.add(new Object[] { spec.query(), name, spec.expectedEventIds(), spec.joinKeys() });
  }
 
  return results;
  }
 
- BaseEqlSpecTestCase(String index, String query, String name, long[] eventIds) {
+ BaseEqlSpecTestCase(String index, String query, String name, long[] eventIds, String[] joinKeys) {
  this.index = index;
 
  this.query = query;
  this.name = name;
  this.eventIds = eventIds;
+ this.joinKeys = joinKeys;
  }
 
  public void test() throws Exception {
@@ -191,6 +197,44 @@ protected void assertSequences(List<Sequence> sequences) {
  .flatMap(s -> s.events().stream())
  .collect(toList());
  assertEvents(events);
+ List<Object> keys = sequences.stream()
+ .flatMap(s -> s.joinKeys().stream())
+ .collect(toList());
+ assertEvents(events);
+ assertJoinKeys(keys);
+ }
+
+ private void assertJoinKeys(List<Object> keys) {
+ logger.debug("Join keys {}", new Object() {
+ public String toString() {
+ return keysToString(keys);
+ }
+ });
+
+ if (joinKeys == null || joinKeys.length == 0) {
+ return;
+ }
+ String[] actual = new String[keys.size()];
+ int i = 0;
+ for (Object key : keys) {
+ if (key == null) {
+ actual[i] = "null";
+ } else {
+ actual[i] = key.toString();
+ }
+ i++;
+ }
+ assertArrayEquals(LoggerMessageFormat.format(null, "unexpected result for spec[{}] [{}] -> {} vs {}", name, query,
+ Arrays.toString(joinKeys), Arrays.toString(actual)), joinKeys, actual);
+ }
+
+ private String keysToString(List<Object> keys) {
+ StringJoiner sj = new StringJoiner(",", "[", "]");
+ for (Object key : keys) {
+ sj.add(key.toString());
+ sj.add("\n");
+ }
+ return sj.toString();
  }
 
  @Override

x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlDateNanosSpecTestCase.java

@@ -21,13 +21,13 @@ public static List<Object[]> readTestSpecs() throws Exception {
  }
 
  // constructor for "local" rest tests
- public EqlDateNanosSpecTestCase(String query, String name, long[] eventIds) {
- this(TEST_NANOS_INDEX, query, name, eventIds);
+ public EqlDateNanosSpecTestCase(String query, String name, long[] eventIds, String[] joinKeys) {
+ this(TEST_NANOS_INDEX, query, name, eventIds, joinKeys);
  }
 
  // constructor for multi-cluster tests
- public EqlDateNanosSpecTestCase(String index, String query, String name, long[] eventIds) {
- super(index, query, name, eventIds);
+ public EqlDateNanosSpecTestCase(String index, String query, String name, long[] eventIds, String[] joinKeys) {
+ super(index, query, name, eventIds, joinKeys);
  }
 
  @Override

x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlExtraSpecTestCase.java

@@ -21,13 +21,13 @@ public static List<Object[]> readTestSpecs() throws Exception {
  }
 
  // constructor for "local" rest tests
- public EqlExtraSpecTestCase(String query, String name, long[] eventIds) {
- this(TEST_EXTRA_INDEX, query, name, eventIds);
+ public EqlExtraSpecTestCase(String query, String name, long[] eventIds, String[] joinKeys) {
+ this(TEST_EXTRA_INDEX, query, name, eventIds, joinKeys);
  }
 
  // constructor for multi-cluster tests
- public EqlExtraSpecTestCase(String index, String query, String name, long[] eventIds) {
- super(index, query, name, eventIds);
+ public EqlExtraSpecTestCase(String index, String query, String name, long[] eventIds, String[] joinKeys) {
+ super(index, query, name, eventIds, joinKeys);
  }
 
  @Override

x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpec.java

@@ -19,6 +19,7 @@ public class EqlSpec {
  private String[] tags;
  private String query;
  private long[] expectedEventIds;
+ private String[] joinKeys;
 
  public String name() {
  return name;
@@ -68,6 +69,14 @@ public void expectedEventIds(long[] expectedEventIds) {
  this.expectedEventIds = expectedEventIds;
  }
 
+ public String[] joinKeys() {
+ return joinKeys;
+ }
+
+ public void joinKeys(String[] joinKeys) {
+ this.joinKeys = joinKeys;
+ }
+
  @Override
  public String toString() {
  String str = "";
@@ -83,6 +92,10 @@ public String toString() {
  if (expectedEventIds != null) {
  str = appendWithComma(str, "expected_event_ids", Arrays.toString(expectedEventIds));
  }
+
+ if (joinKeys != null) {
+ str = appendWithComma(str, "join_keys", Arrays.toString(joinKeys));
+ }
  return str;
  }
 

x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpecLoader.java

@@ -101,6 +101,9 @@ private static List<EqlSpec> readFromStream(InputStream is, Set<String> uniqueTe
  }
  spec.expectedEventIds(expectedEventIds);
  }
+
+ arr = table.getList("join_keys");
+ spec.joinKeys(arr != null ? arr.toArray(new String[0]) : new String[0]);
  validateAndAddSpec(testSpecs, spec, uniqueTestNames);
  }
 

x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpecTestCase.java

@@ -28,12 +28,12 @@ protected String tiebreaker() {
  }
 
  // constructor for "local" rest tests
- public EqlSpecTestCase(String query, String name, long[] eventIds) {
- this(TEST_INDEX, query, name, eventIds);
+ public EqlSpecTestCase(String query, String name, long[] eventIds, String[] joinKeys) {
+ this(TEST_INDEX, query, name, eventIds, joinKeys);
  }
 
  // constructor for multi-cluster tests
- public EqlSpecTestCase(String index, String query, String name, long[] eventIds) {
- super(index, query, name, eventIds);
+ public EqlSpecTestCase(String index, String query, String name, long[] eventIds, String[] joinKeys) {
+ super(index, query, name, eventIds, joinKeys);
  }
 }

x-pack/plugin/eql/qa/common/src/main/resources/additional_test_queries.toml

@@ -430,3 +430,63 @@ query = '''
 process where substring(command_line, 5) regex (".*?net[1]? localgroup.*?", ".*? myappserver.py .*?")
 '''
 
+[[queries]]
+description = "This query has an important meaning when considering its pair - sequenceWithRequiredUserDomain - it shows that when using '?' for a key field, this one becomes null-aware, meaning a null value can be used as a join key"
+name = "sequenceWithOptionalUserDomain"
+expected_event_ids = [1, 57]
+join_keys = ["null"]
+query = '''
+sequence by ?user_domain [process where true] [registry where true]
+'''
+
+[[queries]]
+name = "sequenceWithRequiredUserDomain"
+expected_event_ids = []
+join_keys = []
+query = '''
+sequence by user_domain [process where true] [registry where true]
+'''
+
+[[queries]]
+description = "An equivalent (without optional keys) query can be found in test_queries.toml - twoSequencesWithTwoKeys"
+name = "twoSequencesWithTwoKeys_AndOptionals"
+query = '''
+sequence by ?x
+ [process where true] by unique_pid, process_path, ?z
+ [process where opcode == 1] by unique_ppid, parent_process_path, ?w
+'''
+expected_event_ids = [48, 53,
+ 53, 54,
+ 54, 56,
+ 97, 98]
+join_keys = ["null", "48", "C:\\Python27\\python.exe", "null",
+ "null", "53", "C:\\Windows\\System32\\cmd.exe", "null",
+ "null", "54", "C:\\Python27\\python.exe", "null",
+ "null", "750058", "C:\\Windows\\System32\\net.exe", "null"]
+
+[[queries]]
+name = "sequenceOnOneNullKey"
+query = '''
+sequence
+ [process where parent_process_path == null] by parent_process_path
+ [any where true] by parent_process_path
+'''
+expected_event_ids = []
+
+[[queries]]
+name = "sequenceOnTwoNullKeys"
+query = '''
+sequence by ppid 
+ [process where parent_process_path == null] by parent_process_path
+ [any where true] by parent_process_path
+'''
+expected_event_ids = []
+
+[[queries]]
+name = "sequenceOnImplicitNullKeys"
+query = '''
+sequence by ppid, parent_process_path
+ [process where parent_process_path == null]
+ [any where true]
+'''
+expected_event_ids = []

x-pack/plugin/eql/qa/common/src/main/resources/data/extra.data

@@ -21,18 +21,137 @@
  "@timestamp": "10",
  "event_type": "REQUEST",
  "transID": 1235,
- "sequence": 1
+ "sequence": 4
  },
  {
  "@timestamp": "11",
  "event_type": "ERROR",
  "transID": 1235,
- "sequence": 2
+ "sequence": 5
  },
  {
  "@timestamp": "11",
  "event_type": "STAT",
  "transID": 1235,
- "sequence": 3
+ "sequence": 6
+ },
+ {
+ "@timestamp": "100",
+ "event_type": "OPTIONAL",
+ "optional_field_default_null": null,
+ "sequence": 7
+ },
+ {
+ "@timestamp": "101",
+ "event_type": "OPTIONAL",
+ "sequence": 8,
+ "transID": 1235
+ },
+ {
+ "@timestamp": "102",
+ "event_type": "OPTIONAL",
+ "optional_field_default_null": null,
+ "sequence": 9
+ },
+ {
+ "@timestamp": "1",
+ "event_type": "process",
+ "transID": 1,
+ "process.pid": 123,
+ "sequence": 10
+ },
+ {
+ "@timestamp": "1",
+ "event_type": "process",
+ "transID": 2,
+ "sequence": 11
+ },
+ {
+ "@timestamp": "2",
+ "event_type": "process",
+ "transID": 2,
+ "process.pid": 123,
+ "sequence": 12
+ },
+ {
+ "@timestamp": "3",
+ "event_type": "file",
+ "transID": 0,
+ "process.pid": 123,
+ "sequence": 13
+ },
+ {
+ "@timestamp": "4",
+ "event_type": "file",
+ "transID": 0,
+ "process.pid": 123,
+ "sequence": 14
+ },
+ {
+ "@timestamp": "5",
+ "event_type": "file",
+ "transID": 0,
+ "process.pid": 123,
+ "sequence": 15
+ },
+ {
+ "@timestamp": "6",
+ "event_type": "file",
+ "transID": 0,
+ "sequence": 16
+ },
+ {
+ "@timestamp": "6",
+ "event_type": "file",
+ "transID": 0,
+ "sequence": 17
+ },
+ {
+ "@timestamp": "7",
+ "event_type": "process",
+ "transID": 2,
+ "process.entity_id": 512,
+ "process.pid": 123,
+ "sequence": 18
+ },
+ {
+ "@timestamp": "8",
+ "event_type": "file",
+ "transID": 0,
+ "process.entity_id": 512,
+ "process.pid": 123,
+ "sequence": 19
+ },
+ {
+ "@timestamp": "9",
+ "event_type": "file",
+ "transID": 0,
+ "process.entity_id": 512,
+ "process.pid": 123,
+ "sequence": 20
+ },
+ {
+ "@timestamp": "10",
+ "event_type": "file",
+ "transID": 0,
+ "process.entity_id": 512,
+ "process.pid": 123,
+ "sequence": 21
+ },
+ {
+ "@timestamp": "11",
+ "event_type": "file",
+ "transID": 0,
+ "process.entity_id": 512,
+ "process.pid": 123,
+ "sequence": 22
+ },
+ {
+ "@timestamp": "12",
+ "event_type": "file",
+ "transID": 0,
+ "process.entity_id": 512,
+ "process.pid": 123,
+ "sequence": 23
  }
 ]

x-pack/plugin/eql/qa/common/src/main/resources/data/extra.mapping

@@ -20,6 +20,13 @@
  "path": "sequence"
  }
  }
+ },
+ "optional_field_mapping_only": {
+ "type": "keyword"
+ },
+ "optional_field_default_null": {
+ "type": "keyword",
+ "null_value": "NULL"
  }
  }
 }