Tests and bug fixes

x-pack/plugins/security_solution/server/lib/detection_engine/scripts/create_threat_data.sh

@@ -12,7 +12,7 @@ set -e
 
 # Adds port mock data to a threat list for testing.
 # Example: ./create_threat_data.sh
-# Example: ./create_threat_data.sh 1000 2000
+# Example: ./create_threat_data.sh 1 500
 
 START=${1:-1}
 END=${2:-1000}
@@ -22,7 +22,7 @@ do {
 curl -s -k \
  -H "Content-Type: application/json" \
  -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \
- -X PUT ${ELASTICSEARCH_URL}/mock-threat-list/_doc/$i \
+ -X PUT ${ELASTICSEARCH_URL}/mock-threat-list-1/_doc/$i \
  --data "
 {
  \"@timestamp\": \"$(date -u +"%Y-%m-%dT%H:%M:%SZ")\",

x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/query_with_threat_mapping_perf.json

@@ -1,15 +1,17 @@
 {
- "concurrent_searches": 100,
+ "concurrent_searches": 10,
  "items_per_search": 10,
- "name": "Threat Indicator Concurrent Searches",
+ "index": ["auditbeat-*", "endgame-*", "filebeat-*", "logs-*", "packetbeat-*", "winlogbeat-*"],
+ "name": "Indicator Match Concurrent Searches",
  "description": "Does 100 Concurrent searches with 10 items per search",
- "rule_id": "threat_mapping_concurrent_search",
+ "rule_id": "indicator_concurrent_search",
  "risk_score": 1,
  "severity": "high",
  "type": "threat_match",
  "query": "*:*",
  "tags": ["concurrent_searches_test", "from_script"],
- "threat_index": ["mock-threat-list"],
+ "threat_index": ["mock-threat-list-1"],
+ "threat_language": "kuery",
  "threat_query": "*:*",
  "threat_mapping": [
  {

x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts

@@ -504,7 +504,7 @@ describe('rules_notification_alert_type', () => {
  await alert.executor(payload);
  expect(logger.error).toHaveBeenCalled();
  expect(logger.error.mock.calls[0][0]).toContain(
- 'An error occurred during rule execution: message: "Threat Match rule is missing threatQuery and/or threatIndex and/or threatMapping: threatQuery: "undefined" threatIndex: "undefined" threatMapping: "undefined"" name: "Detect Root/Admin Users" id: "04128c15-0d1b-4716-a4c5-46997ac7f3bd" rule id: "rule-1" signals index: ".siem-signals"'
+ 'An error occurred during rule execution: message: "Indicator match is missing threatQuery and/or threatIndex and/or threatMapping: threatQuery: "undefined" threatIndex: "undefined" threatMapping: "undefined"" name: "Detect Root/Admin Users" id: "04128c15-0d1b-4716-a4c5-46997ac7f3bd" rule id: "rule-1" signals index: ".siem-signals"'
  );
  });
  });

x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts

@@ -9,7 +9,6 @@ import { buildThreatMappingFilter } from './build_threat_mapping_filter';
 import { getFilter } from '../get_filter';
 import { searchAfterAndBulkCreate } from '../search_after_bulk_create';
 import { CreateThreatSignalOptions } from './types';
-import { combineResults } from './utils';
 import { SearchAfterAndBulkCreateReturnType } from '../types';
 
 export const createThreatSignal = async ({
@@ -77,7 +76,7 @@ export const createThreatSignal = async ({
  `${threatFilter.query.bool.should.length} indicator items are being checked for existence of matches`
  )
  );
- const newResult = await searchAfterAndBulkCreate({
+ const result = await searchAfterAndBulkCreate({
  gap,
  previousStartedAt,
  listClient,
@@ -104,16 +103,15 @@ export const createThreatSignal = async ({
  throttle,
  buildRuleMessage,
  });
- const results = combineResults(currentResult, newResult);
  logger.debug(
  buildRuleMessage(
  `${
  threatFilter.query.bool.should.length
- } items have completed match checks and the total time to search was ${
- newResult.searchAfterTimes.length !== 0 ? newResult.searchAfterTimes : '(unknown) '
+ } items have completed match checks and the total times to search were ${
+ result.searchAfterTimes.length !== 0 ? result.searchAfterTimes : '(unknown) '
  }ms`
  )
  );
- return results;
+ return result;
  }
 };

x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts

@@ -50,7 +50,7 @@ export const createThreatSignals = async ({
  concurrentSearches,
  itemsPerSearch,
 }: CreateThreatSignalsOptions): Promise<SearchAfterAndBulkCreateReturnType> => {
- logger.debug(buildRuleMessage('Indicator matching starting'));
+ logger.debug(buildRuleMessage('Indicator matching rule starting'));
  const perPage = concurrentSearches * itemsPerSearch;
 
  let results: SearchAfterAndBulkCreateReturnType = {
@@ -70,7 +70,7 @@ export const createThreatSignals = async ({
  language: threatLanguage,
  index: threatIndex,
  });
- logger.debug(buildRuleMessage(`Total indicator items are ${threatListCount}`));
+ logger.debug(buildRuleMessage(`Total indicator items: ${threatListCount}`));
 
  let threatList = await getThreatList({
  callCluster: services.callCluster,
@@ -88,14 +88,13 @@ export const createThreatSignals = async ({
  perPage,
  });
 
- const chunks = chunk(itemsPerSearch, threatList.hits.hits);
- logger.debug(
- buildRuleMessage(
- `${chunks.length} concurrent indicator searches are starting. Each search has ${itemsPerSearch} indicator items`
- )
- );
-
  while (threatList.hits.hits.length !== 0) {
+ const chunks = chunk(itemsPerSearch, threatList.hits.hits);
+ logger.debug(
+ buildRuleMessage(
+ `${chunks.length} concurrent indicator searches are starting. Each search can have up to ${itemsPerSearch} indicator items per search`
+ )
+ );
  const concurrentSearchesPerformed = chunks.map<Promise<SearchAfterAndBulkCreateReturnType>>(
  (slicedChunk) =>
  createThreatSignal({
@@ -171,6 +170,6 @@ export const createThreatSignals = async ({
  });
  }
 
- logger.debug(buildRuleMessage('Indicator Matching completed'));
+ logger.debug(buildRuleMessage('Indicator matching rule has completed'));
  return results;
 };