From fb8cd5b0916327952e6d053bc19d7c2ea9ebf77a Mon Sep 17 00:00:00 2001 From: Frank Hassanabad Date: Fri, 6 Nov 2020 20:46:52 -0700 Subject: [PATCH] [Security Solution] Adds logging and performance fan out API for threat/Indicator matching (#82546) ## Summary * Adds logging output for trouble shooting * Adds an API to be able to configure how many concurrent searches and how many items per search to use API additions are these two switches: ``` concurrent_searches items_per_search ``` When you create a rule. You can use the following example to post one or to change the settings to see the performance impact: ```ts ./post_rule.sh ./rules/queries/query_with_threat_mapping_perf.json ``` Without using these two experimental API settings, the functionality is the same as the existing algorithm and only advanced users will be able to set the additional REST settings through this API. If you use the front end after setting the settings, the settings will be reset as that's how the forms code currently works and this will not preserve the settings if afterwards a rule is edited/changed. Both these API settings should be considered experimental and potentially breakable as we figure out the best performance strategies for indicator matching. ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --- .../request/add_prepackaged_rules_schema.ts | 4 + .../request/create_rules_schema.test.ts | 18 + .../schemas/request/create_rules_schema.ts | 4 + .../create_rules_type_dependents.test.ts | 32 ++ .../request/create_rules_type_dependents.ts | 12 +- .../schemas/request/import_rules_schema.ts | 4 + .../schemas/request/patch_rules_schema.ts | 4 + .../schemas/request/update_rules_schema.ts | 4 + .../schemas/response/rules_schema.test.ts | 4 +- .../schemas/response/rules_schema.ts | 10 + .../schemas/types/threat_mapping.test.ts | 58 ++- .../schemas/types/threat_mapping.ts | 11 + .../rules/step_define_rule/translations.tsx | 2 +- .../routes/__mocks__/request_responses.ts | 2 + .../routes/rules/create_rules_bulk_route.ts | 4 + .../routes/rules/create_rules_route.ts | 4 + .../routes/rules/import_rules_route.ts | 6 + .../routes/rules/patch_rules_bulk_route.ts | 4 + .../routes/rules/patch_rules_route.ts | 4 + .../routes/rules/update_rules_bulk_route.ts | 4 + .../routes/rules/update_rules_route.ts | 4 + .../detection_engine/routes/rules/utils.ts | 2 + .../rules/create_rules.mock.ts | 4 + .../detection_engine/rules/create_rules.ts | 4 + .../rules/install_prepacked_rules.ts | 4 + .../rules/patch_rules.mock.ts | 4 + .../lib/detection_engine/rules/patch_rules.ts | 6 + .../lib/detection_engine/rules/types.ts | 8 + .../rules/update_prepacked_rules.ts | 4 + .../rules/update_rules.mock.ts | 4 + .../detection_engine/rules/update_rules.ts | 4 + .../lib/detection_engine/rules/utils.test.ts | 6 + .../lib/detection_engine/rules/utils.ts | 4 + .../scripts/create_threat_data.sh | 4 +- .../query_with_threat_mapping_perf.json | 32 ++ .../signals/__mocks__/es_results.ts | 2 + .../signals/signal_params_schema.ts | 2 + .../signals/signal_rule_alert_type.test.ts | 2 +- .../signals/signal_rule_alert_type.ts | 6 +- .../build_threat_mapping_filter.test.ts | 69 ++-- .../build_threat_mapping_filter.ts | 41 +- .../threat_mapping/create_threat_signal.ts | 51 +-- .../threat_mapping/create_threat_signals.ts | 133 ++++-- .../signals/threat_mapping/get_threat_list.ts | 37 ++ .../signals/threat_mapping/types.ts | 31 +- .../signals/threat_mapping/utils.test.ts | 387 +++++++++++++++++- .../signals/threat_mapping/utils.ts | 71 ++++ .../server/lib/detection_engine/types.ts | 4 + 48 files changed, 981 insertions(+), 144 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/query_with_threat_mapping_perf.json diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.ts index 6ffbf4e4c8d4ca..1b0417cf59bc28 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/add_prepackaged_rules_schema.ts @@ -48,6 +48,8 @@ import { } from '../common/schemas'; import { threat_index, + concurrent_searches, + items_per_search, threat_query, threat_filters, threat_mapping, @@ -130,6 +132,8 @@ export const addPrepackagedRulesSchema = t.intersection([ threat_query, // defaults to "undefined" if not set during decode threat_index, // defaults to "undefined" if not set during decode threat_language, // defaults "undefined" if not set during decode + concurrent_searches, // defaults to "undefined" if not set during decode + items_per_search, // defaults to "undefined" if not set during decode }) ), ]); diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.test.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.test.ts index a4f002b589ef55..1b6a8d6f27762e 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.test.ts @@ -1702,5 +1702,23 @@ describe('create rules schema', () => { expect(getPaths(left(message.errors))).toEqual([]); expect(message.schema).toEqual(expected); }); + + test('You can set a threat query, index, mapping, filters, concurrent_searches, items_per_search with a when creating a rule', () => { + const payload: CreateRulesSchema = { + ...getCreateThreatMatchRulesSchemaMock(), + concurrent_searches: 10, + items_per_search: 10, + }; + const decoded = createRulesSchema.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + const expected: CreateRulesSchemaDecoded = { + ...getCreateThreatMatchRulesSchemaDecodedMock(), + concurrent_searches: 10, + items_per_search: 10, + }; + expect(getPaths(left(message.errors))).toEqual([]); + expect(message.schema).toEqual(expected); + }); }); }); diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.ts index d8e7614fcb8401..2fe52bbe470a52 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_schema.ts @@ -49,6 +49,8 @@ import { } from '../common/schemas'; import { threat_index, + concurrent_searches, + items_per_search, threat_query, threat_filters, threat_mapping, @@ -126,6 +128,8 @@ export const createRulesSchema = t.intersection([ threat_filters, // defaults to "undefined" if not set during decode threat_index, // defaults to "undefined" if not set during decode threat_language, // defaults "undefined" if not set during decode + concurrent_searches, // defaults "undefined" if not set during decode + items_per_search, // defaults "undefined" if not set during decode }) ), ]); diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_type_dependents.test.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_type_dependents.test.ts index 75ad92578318ce..a78b41cd0da187 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_type_dependents.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_type_dependents.test.ts @@ -125,4 +125,36 @@ describe('create_rules_type_dependents', () => { const errors = createRuleValidateTypeDependents(schema); expect(errors).toEqual([]); }); + + test('validates that both "items_per_search" and "concurrent_searches" works when together', () => { + const schema: CreateRulesSchema = { + ...getCreateThreatMatchRulesSchemaMock(), + concurrent_searches: 10, + items_per_search: 10, + }; + const errors = createRuleValidateTypeDependents(schema); + expect(errors).toEqual([]); + }); + + test('does NOT validate when only "items_per_search" is present', () => { + const schema: CreateRulesSchema = { + ...getCreateThreatMatchRulesSchemaMock(), + items_per_search: 10, + }; + const errors = createRuleValidateTypeDependents(schema); + expect(errors).toEqual([ + 'when "items_per_search" exists, "concurrent_searches" must also exist', + ]); + }); + + test('does NOT validate when only "concurrent_searches" is present', () => { + const schema: CreateRulesSchema = { + ...getCreateThreatMatchRulesSchemaMock(), + concurrent_searches: 10, + }; + const errors = createRuleValidateTypeDependents(schema); + expect(errors).toEqual([ + 'when "concurrent_searches" exists, "items_per_search" must also exist', + ]); + }); }); diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_type_dependents.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_type_dependents.ts index c2a41005ebf4d6..c93b0f0b14f6ad 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_type_dependents.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_rules_type_dependents.ts @@ -110,17 +110,23 @@ export const validateThreshold = (rule: CreateRulesSchema): string[] => { export const validateThreatMapping = (rule: CreateRulesSchema): string[] => { let errors: string[] = []; if (isThreatMatchRule(rule.type)) { - if (!rule.threat_mapping) { + if (rule.threat_mapping == null) { errors = ['when "type" is "threat_match", "threat_mapping" is required', ...errors]; } else if (rule.threat_mapping.length === 0) { errors = ['threat_mapping" must have at least one element', ...errors]; } - if (!rule.threat_query) { + if (rule.threat_query == null) { errors = ['when "type" is "threat_match", "threat_query" is required', ...errors]; } - if (!rule.threat_index) { + if (rule.threat_index == null) { errors = ['when "type" is "threat_match", "threat_index" is required', ...errors]; } + if (rule.concurrent_searches == null && rule.items_per_search != null) { + errors = ['when "items_per_search" exists, "concurrent_searches" must also exist', ...errors]; + } + if (rule.concurrent_searches != null && rule.items_per_search == null) { + errors = ['when "concurrent_searches" exists, "items_per_search" must also exist', ...errors]; + } } return errors; }; diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.ts index 852394b74767b7..4f28c469238658 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/import_rules_schema.ts @@ -55,6 +55,8 @@ import { } from '../common/schemas'; import { threat_index, + items_per_search, + concurrent_searches, threat_query, threat_filters, threat_mapping, @@ -149,6 +151,8 @@ export const importRulesSchema = t.intersection([ threat_query, // defaults to "undefined" if not set during decode threat_index, // defaults to "undefined" if not set during decode threat_language, // defaults "undefined" if not set during decode + concurrent_searches, // defaults to "undefined" if not set during decode + items_per_search, // defaults to "undefined" if not set during decode }) ), ]); diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_schema.ts index f4dce5c7ac05f8..45fcfbaa3c76aa 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/patch_rules_schema.ts @@ -50,6 +50,8 @@ import { } from '../common/schemas'; import { threat_index, + concurrent_searches, + items_per_search, threat_query, threat_filters, threat_mapping, @@ -109,6 +111,8 @@ export const patchRulesSchema = t.exact( threat_filters, threat_mapping, threat_language, + concurrent_searches, + items_per_search, }) ); diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_schema.ts index b0cd8b1c53688a..5d759fc12cd528 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/update_rules_schema.ts @@ -51,6 +51,8 @@ import { } from '../common/schemas'; import { threat_index, + concurrent_searches, + items_per_search, threat_query, threat_filters, threat_mapping, @@ -134,6 +136,8 @@ export const updateRulesSchema = t.intersection([ threat_filters, // defaults to "undefined" if not set during decode threat_index, // defaults to "undefined" if not set during decode threat_language, // defaults "undefined" if not set during decode + concurrent_searches, // defaults to "undefined" if not set during decode + items_per_search, // defaults to "undefined" if not set during decode }) ), ]); diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.test.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.test.ts index 82675768a11b7c..3508526e182d77 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.test.ts @@ -762,9 +762,9 @@ describe('rules_schema', () => { expect(fields).toEqual(expected); }); - test('should return 5 fields for a rule of type "threat_match"', () => { + test('should return 8 fields for a rule of type "threat_match"', () => { const fields = addThreatMatchFields({ type: 'threat_match' }); - expect(fields.length).toEqual(6); + expect(fields.length).toEqual(8); }); }); diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts index e85beddf0e51e9..0f7d04763a36f1 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/response/rules_schema.ts @@ -63,6 +63,8 @@ import { } from '../common/schemas'; import { threat_index, + concurrent_searches, + items_per_search, threat_query, threat_filters, threat_mapping, @@ -144,6 +146,8 @@ export const dependentRulesSchema = t.partial({ threat_filters, threat_index, threat_query, + concurrent_searches, + items_per_search, threat_mapping, threat_language, }); @@ -282,6 +286,12 @@ export const addThreatMatchFields = (typeAndTimelineOnly: TypeAndTimelineOnly): t.exact(t.partial({ threat_language: dependentRulesSchema.props.threat_language })), t.exact(t.partial({ threat_filters: dependentRulesSchema.props.threat_filters })), t.exact(t.partial({ saved_id: dependentRulesSchema.props.saved_id })), + t.exact(t.partial({ concurrent_searches: dependentRulesSchema.props.concurrent_searches })), + t.exact( + t.partial({ + items_per_search: dependentRulesSchema.props.items_per_search, + }) + ), ]; } else { return []; diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/types/threat_mapping.test.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/types/threat_mapping.test.ts index 63d593ea84e67f..d8f61e4309b17a 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/types/threat_mapping.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/types/threat_mapping.test.ts @@ -5,6 +5,8 @@ */ import { + concurrent_searches, + items_per_search, ThreatMapping, threatMappingEntries, ThreatMappingEntries, @@ -33,7 +35,7 @@ describe('threat_mapping', () => { expect(message.schema).toEqual(payload); }); - test('it should NOT validate an extra entry item', () => { + test('it should fail validation with an extra entry item', () => { const payload: ThreatMappingEntries & Array<{ extra: string }> = [ { field: 'field.one', @@ -50,7 +52,7 @@ describe('threat_mapping', () => { expect(message.schema).toEqual({}); }); - test('it should NOT validate a non string', () => { + test('it should fail validation with a non string', () => { const payload = ([ { field: 5, @@ -66,7 +68,7 @@ describe('threat_mapping', () => { expect(message.schema).toEqual({}); }); - test('it should NOT validate a wrong type', () => { + test('it should fail validation with a wrong type', () => { const payload = ([ { field: 'field.one', @@ -107,7 +109,7 @@ describe('threat_mapping', () => { }); }); - test('it should NOT validate an extra key', () => { + test('it should fail validate with an extra key', () => { const payload: ThreatMapping & Array<{ extra: string }> = [ { entries: [ @@ -129,7 +131,7 @@ describe('threat_mapping', () => { expect(message.schema).toEqual({}); }); - test('it should NOT validate an extra inner entry', () => { + test('it should fail validate with an extra inner entry', () => { const payload: ThreatMapping & Array<{ entries: Array<{ extra: string }> }> = [ { entries: [ @@ -151,7 +153,7 @@ describe('threat_mapping', () => { expect(message.schema).toEqual({}); }); - test('it should NOT validate an extra inner entry with the wrong data type', () => { + test('it should fail validate with an extra inner entry with the wrong data type', () => { const payload = ([ { entries: [ @@ -173,4 +175,48 @@ describe('threat_mapping', () => { ]); expect(message.schema).toEqual({}); }); + + test('it should fail validation when concurrent_searches is < 0', () => { + const payload = -1; + const decoded = concurrent_searches.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual([ + 'Invalid value "-1" supplied to "PositiveIntegerGreaterThanZero"', + ]); + expect(message.schema).toEqual({}); + }); + + test('it should fail validation when concurrent_searches is 0', () => { + const payload = 0; + const decoded = concurrent_searches.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual([ + 'Invalid value "0" supplied to "PositiveIntegerGreaterThanZero"', + ]); + expect(message.schema).toEqual({}); + }); + + test('it should fail validation when items_per_search is 0', () => { + const payload = 0; + const decoded = items_per_search.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual([ + 'Invalid value "0" supplied to "PositiveIntegerGreaterThanZero"', + ]); + expect(message.schema).toEqual({}); + }); + + test('it should fail validation when items_per_search is < 0', () => { + const payload = -1; + const decoded = items_per_search.decode(payload); + const checked = exactCheck(payload, decoded); + const message = pipe(checked, foldLeftRight); + expect(getPaths(left(message.errors))).toEqual([ + 'Invalid value "-1" supplied to "PositiveIntegerGreaterThanZero"', + ]); + expect(message.schema).toEqual({}); + }); }); diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/types/threat_mapping.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/types/threat_mapping.ts index a1be6485f596b7..dec8ddd0001324 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/schemas/types/threat_mapping.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/types/threat_mapping.ts @@ -9,6 +9,7 @@ import * as t from 'io-ts'; import { language } from '../common/schemas'; import { NonEmptyString } from './non_empty_string'; +import { PositiveIntegerGreaterThanZero } from './positive_integer_greater_than_zero'; export const threat_query = t.string; export type ThreatQuery = t.TypeOf; @@ -55,3 +56,13 @@ export const threat_language = t.union([language, t.undefined]); export type ThreatLanguage = t.TypeOf; export const threatLanguageOrUndefined = t.union([threat_language, t.undefined]); export type ThreatLanguageOrUndefined = t.TypeOf; + +export const concurrent_searches = PositiveIntegerGreaterThanZero; +export type ConcurrentSearches = t.TypeOf; +export const concurrentSearchesOrUndefined = t.union([concurrent_searches, t.undefined]); +export type ConcurrentSearchesOrUndefined = t.TypeOf; + +export const items_per_search = PositiveIntegerGreaterThanZero; +export type ItemsPerSearch = t.TypeOf; +export const itemsPerSearchOrUndefined = t.union([items_per_search, t.undefined]); +export type ItemsPerSearchOrUndefined = t.TypeOf; diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/translations.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/translations.tsx index 164b1df8463e6f..221963767caade 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/translations.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/translations.tsx @@ -95,7 +95,7 @@ export const THREAT_MATCH_INDEX_HELPER_TEXT = i18n.translate( export const THREAT_MATCH_REQUIRED = i18n.translate( 'xpack.securitySolution.detectionEngine.createRule.stepDefineRule.customThreatQueryFieldRequiredError', { - defaultMessage: 'At least one threat match is required.', + defaultMessage: 'At least one indicator match is required.', } ); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts index 94b820344b37ca..773e84d9c88fce 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts @@ -407,6 +407,8 @@ export const getResult = (): RuleAlertType => ({ note: '# Investigative notes', version: 1, exceptionsList: getListArrayMock(), + concurrentSearches: undefined, + itemsPerSearch: undefined, }, createdAt: new Date('2019-12-13T16:40:33.400Z'), updatedAt: new Date('2019-12-13T16:40:33.400Z'), diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_bulk_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_bulk_route.ts index 8c7a19869ce188..aa409580df9655 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_bulk_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_bulk_route.ts @@ -102,6 +102,8 @@ export const createRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) => threat_mapping: threatMapping, threat_query: threatQuery, threat_language: threatLanguage, + concurrent_searches: concurrentSearches, + items_per_search: itemsPerSearch, threshold, throttle, timestamp_override: timestampOverride, @@ -193,6 +195,8 @@ export const createRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) => threatQuery, threatIndex, threatLanguage, + concurrentSearches, + itemsPerSearch, threshold, timestampOverride, references, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts index 6ba7bc78fbded6..97c05b4626ddc2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts @@ -85,6 +85,8 @@ export const createRulesRoute = (router: IRouter, ml: SetupPlugins['ml']): void threat_query: threatQuery, threat_mapping: threatMapping, threat_language: threatLanguage, + concurrent_searches: concurrentSearches, + items_per_search: itemsPerSearch, throttle, timestamp_override: timestampOverride, to, @@ -182,6 +184,8 @@ export const createRulesRoute = (router: IRouter, ml: SetupPlugins['ml']): void threatQuery, threatMapping, threatLanguage, + concurrentSearches, + itemsPerSearch, timestampOverride, references, note, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/import_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/import_rules_route.ts index 7cbcf25590921c..688036c59c8ffe 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/import_rules_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/import_rules_route.ts @@ -169,6 +169,8 @@ export const importRulesRoute = (router: IRouter, config: ConfigType, ml: SetupP threat_query: threatQuery, threat_mapping: threatMapping, threat_language: threatLanguage, + concurrent_searches: concurrentSearches, + items_per_search: itemsPerSearch, threshold, timestamp_override: timestampOverride, to, @@ -235,6 +237,8 @@ export const importRulesRoute = (router: IRouter, config: ConfigType, ml: SetupP threatQuery, threatMapping, threatLanguage, + concurrentSearches, + itemsPerSearch, timestampOverride, references, note, @@ -284,6 +288,8 @@ export const importRulesRoute = (router: IRouter, config: ConfigType, ml: SetupP threatQuery, threatMapping, threatLanguage, + concurrentSearches, + itemsPerSearch, references, note, version, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_bulk_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_bulk_route.ts index 4c310774ec72b2..7dfb4daa1a0a22 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_bulk_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_bulk_route.ts @@ -97,6 +97,8 @@ export const patchRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) => threat_query: threatQuery, threat_mapping: threatMapping, threat_language: threatLanguage, + concurrent_searches: concurrentSearches, + items_per_search: itemsPerSearch, timestamp_override: timestampOverride, throttle, references, @@ -162,6 +164,8 @@ export const patchRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) => threatQuery, threatMapping, threatLanguage, + concurrentSearches, + itemsPerSearch, timestampOverride, references, note, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_route.ts index dbdcd9844c0a79..aadb13ef54e726 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_route.ts @@ -83,6 +83,8 @@ export const patchRulesRoute = (router: IRouter, ml: SetupPlugins['ml']) => { threat_query: threatQuery, threat_mapping: threatMapping, threat_language: threatLanguage, + concurrent_searches: concurrentSearches, + items_per_search: itemsPerSearch, timestamp_override: timestampOverride, throttle, references, @@ -161,6 +163,8 @@ export const patchRulesRoute = (router: IRouter, ml: SetupPlugins['ml']) => { threatQuery, threatMapping, threatLanguage, + concurrentSearches, + itemsPerSearch, timestampOverride, references, note, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_bulk_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_bulk_route.ts index b93b3f319193f4..f4a31c2bb456dd 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_bulk_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_bulk_route.ts @@ -102,6 +102,8 @@ export const updateRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) => threat_query: threatQuery, threat_mapping: threatMapping, threat_language: threatLanguage, + concurrent_searches: concurrentSearches, + items_per_search: itemsPerSearch, throttle, timestamp_override: timestampOverride, references, @@ -174,6 +176,8 @@ export const updateRulesBulkRoute = (router: IRouter, ml: SetupPlugins['ml']) => threatQuery, threatMapping, threatLanguage, + concurrentSearches, + itemsPerSearch, timestampOverride, references, note, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_route.ts index ea19fed5d6668b..7ad525b67f7aa1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_route.ts @@ -86,6 +86,8 @@ export const updateRulesRoute = (router: IRouter, ml: SetupPlugins['ml']) => { threat_query: threatQuery, threat_mapping: threatMapping, threat_language: threatLanguage, + concurrent_searches: concurrentSearches, + items_per_search: itemsPerSearch, throttle, timestamp_override: timestampOverride, references, @@ -163,6 +165,8 @@ export const updateRulesRoute = (router: IRouter, ml: SetupPlugins['ml']) => { threatQuery, threatMapping, threatLanguage, + concurrentSearches, + itemsPerSearch, timestampOverride, references, note, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.ts index fb4ba855f65369..7360dc77aac22c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.ts @@ -151,6 +151,8 @@ export const transformAlertToRule = ( threat_query: alert.params.threatQuery, threat_mapping: alert.params.threatMapping, threat_language: alert.params.threatLanguage, + concurrent_searches: alert.params.concurrentSearches, + items_per_search: alert.params.itemsPerSearch, throttle: ruleActions?.ruleThrottle || 'no_actions', timestamp_override: alert.params.timestampOverride, note: alert.params.note, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.mock.ts index 271b1043ea5683..68199c531a2fec 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.mock.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.mock.ts @@ -43,6 +43,8 @@ export const getCreateRulesOptionsMock = (): CreateRulesOptions => ({ threatFilters: undefined, threatMapping: undefined, threatLanguage: undefined, + concurrentSearches: undefined, + itemsPerSearch: undefined, threatQuery: undefined, threatIndex: undefined, threshold: undefined, @@ -94,6 +96,8 @@ export const getCreateMlRulesOptionsMock = (): CreateRulesOptions => ({ threatMapping: undefined, threatQuery: undefined, threatLanguage: undefined, + concurrentSearches: undefined, + itemsPerSearch: undefined, threshold: undefined, timestampOverride: undefined, to: 'now', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.ts index 776882d0f84941..3c814ce7e66067 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/create_rules.ts @@ -46,6 +46,8 @@ export const createRules = async ({ threatFilters, threatIndex, threatLanguage, + concurrentSearches, + itemsPerSearch, threatQuery, threatMapping, threshold, @@ -96,6 +98,8 @@ export const createRules = async ({ threatFilters, threatIndex, threatQuery, + concurrentSearches, + itemsPerSearch, threatMapping, threatLanguage, timestampOverride, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/install_prepacked_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/install_prepacked_rules.ts index 0a43c652234d09..4c01318f02cdea 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/install_prepacked_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/install_prepacked_rules.ts @@ -51,6 +51,8 @@ export const installPrepackagedRules = ( threat_filters: threatFilters, threat_mapping: threatMapping, threat_language: threatLanguage, + concurrent_searches: concurrentSearches, + items_per_search: itemsPerSearch, threat_query: threatQuery, threat_index: threatIndex, threshold, @@ -103,6 +105,8 @@ export const installPrepackagedRules = ( threatFilters, threatMapping, threatLanguage, + concurrentSearches, + itemsPerSearch, threatQuery, threatIndex, threshold, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.mock.ts index ef7cd35f28f1bd..60f1d599470e3c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.mock.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.mock.ts @@ -154,6 +154,8 @@ export const getPatchRulesOptionsMock = (): PatchRulesOptions => ({ threatQuery: undefined, threatMapping: undefined, threatLanguage: undefined, + concurrentSearches: undefined, + itemsPerSearch: undefined, timestampOverride: undefined, to: 'now', type: 'query', @@ -203,6 +205,8 @@ export const getPatchMlRulesOptionsMock = (): PatchRulesOptions => ({ threatQuery: undefined, threatMapping: undefined, threatLanguage: undefined, + concurrentSearches: undefined, + itemsPerSearch: undefined, timestampOverride: undefined, to: 'now', type: 'machine_learning', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.ts index 1982dcf9dd9b67..22b2593283696d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/patch_rules.ts @@ -49,6 +49,8 @@ export const patchRules = async ({ threatQuery, threatMapping, threatLanguage, + concurrentSearches, + itemsPerSearch, timestampOverride, to, type, @@ -97,6 +99,8 @@ export const patchRules = async ({ threatQuery, threatMapping, threatLanguage, + concurrentSearches, + itemsPerSearch, timestampOverride, to, type, @@ -141,6 +145,8 @@ export const patchRules = async ({ threatQuery, threatMapping, threatLanguage, + concurrentSearches, + itemsPerSearch, timestampOverride, to, type, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts index fb4763a982f43f..f6ab3fb0c3ed2e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts @@ -92,6 +92,8 @@ import { ThreatMappingOrUndefined, ThreatFiltersOrUndefined, ThreatLanguageOrUndefined, + ConcurrentSearchesOrUndefined, + ItemsPerSearchOrUndefined, } from '../../../../common/detection_engine/schemas/types/threat_mapping'; import { AlertsClient, PartialAlert } from '../../../../../alerts/server'; @@ -234,6 +236,8 @@ export interface CreateRulesOptions { threatIndex: ThreatIndexOrUndefined; threatQuery: ThreatQueryOrUndefined; threatMapping: ThreatMappingOrUndefined; + concurrentSearches: ConcurrentSearchesOrUndefined; + itemsPerSearch: ItemsPerSearchOrUndefined; threatLanguage: ThreatLanguageOrUndefined; timestampOverride: TimestampOverrideOrUndefined; to: To; @@ -284,6 +288,8 @@ export interface UpdateRulesOptions { threatIndex: ThreatIndexOrUndefined; threatQuery: ThreatQueryOrUndefined; threatMapping: ThreatMappingOrUndefined; + itemsPerSearch: ItemsPerSearchOrUndefined; + concurrentSearches: ConcurrentSearchesOrUndefined; threatLanguage: ThreatLanguageOrUndefined; timestampOverride: TimestampOverrideOrUndefined; to: To; @@ -327,6 +333,8 @@ export interface PatchRulesOptions { severityMapping: SeverityMappingOrUndefined; tags: TagsOrUndefined; threat: ThreatOrUndefined; + itemsPerSearch: ItemsPerSearchOrUndefined; + concurrentSearches: ConcurrentSearchesOrUndefined; threshold: ThresholdOrUndefined; threatFilters: ThreatFiltersOrUndefined; threatIndex: ThreatIndexOrUndefined; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_prepacked_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_prepacked_rules.ts index c685c4198c1193..3d4b27b74c0af0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_prepacked_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_prepacked_rules.ts @@ -52,6 +52,8 @@ export const updatePrepackagedRules = async ( threat_query: threatQuery, threat_mapping: threatMapping, threat_language: threatLanguage, + concurrent_searches: concurrentSearches, + items_per_search: itemsPerSearch, timestamp_override: timestampOverride, references, version, @@ -107,6 +109,8 @@ export const updatePrepackagedRules = async ( threatQuery, threatMapping, threatLanguage, + concurrentSearches, + itemsPerSearch, references, version, note, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.mock.ts index a33651580ef221..34be0f6ad843dc 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.mock.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.mock.ts @@ -49,6 +49,8 @@ export const getUpdateRulesOptionsMock = (): UpdateRulesOptions => ({ threatMapping: undefined, threatLanguage: undefined, timestampOverride: undefined, + concurrentSearches: undefined, + itemsPerSearch: undefined, to: 'now', type: 'query', references: ['http://www.example.com'], @@ -99,6 +101,8 @@ export const getUpdateMlRulesOptionsMock = (): UpdateRulesOptions => ({ threatMapping: undefined, threatLanguage: undefined, timestampOverride: undefined, + concurrentSearches: undefined, + itemsPerSearch: undefined, to: 'now', type: 'machine_learning', references: ['http://www.example.com'], diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.ts index 3da921ed47f26e..5168affca5c624 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/update_rules.ts @@ -50,6 +50,8 @@ export const updateRules = async ({ threatQuery, threatMapping, threatLanguage, + concurrentSearches, + itemsPerSearch, timestampOverride, to, type, @@ -99,6 +101,8 @@ export const updateRules = async ({ threatQuery, threatMapping, threatLanguage, + concurrentSearches, + itemsPerSearch, timestampOverride, to, type, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.test.ts index 654383ff97c7ad..8555af424ecd79 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.test.ts @@ -60,6 +60,8 @@ describe('utils', () => { threatQuery: undefined, threatMapping: undefined, threatLanguage: undefined, + concurrentSearches: undefined, + itemsPerSearch: undefined, to: undefined, timestampOverride: undefined, type: undefined, @@ -108,6 +110,8 @@ describe('utils', () => { threatQuery: undefined, threatMapping: undefined, threatLanguage: undefined, + concurrentSearches: undefined, + itemsPerSearch: undefined, to: undefined, timestampOverride: undefined, type: undefined, @@ -158,6 +162,8 @@ describe('utils', () => { threatLanguage: undefined, to: undefined, timestampOverride: undefined, + concurrentSearches: undefined, + itemsPerSearch: undefined, type: undefined, references: undefined, version: undefined, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.ts index a9a100543b528a..83d9e3fd3e59f8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/utils.ts @@ -43,6 +43,8 @@ import { } from '../../../../common/detection_engine/schemas/common/schemas'; import { PartialFilter } from '../types'; import { + ConcurrentSearchesOrUndefined, + ItemsPerSearchOrUndefined, ListArrayOrUndefined, ThreatFiltersOrUndefined, ThreatIndexOrUndefined, @@ -98,6 +100,8 @@ export interface UpdateProperties { threatQuery: ThreatQueryOrUndefined; threatMapping: ThreatMappingOrUndefined; threatLanguage: ThreatLanguageOrUndefined; + concurrentSearches: ConcurrentSearchesOrUndefined; + itemsPerSearch: ItemsPerSearchOrUndefined; timestampOverride: TimestampOverrideOrUndefined; to: ToOrUndefined; type: TypeOrUndefined; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/create_threat_data.sh b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/create_threat_data.sh index 23c1914387c44d..4807afd71e8d2c 100755 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/create_threat_data.sh +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/create_threat_data.sh @@ -12,7 +12,7 @@ set -e # Adds port mock data to a threat list for testing. # Example: ./create_threat_data.sh -# Example: ./create_threat_data.sh 1000 2000 +# Example: ./create_threat_data.sh 1 500 START=${1:-1} END=${2:-1000} @@ -22,7 +22,7 @@ do { curl -s -k \ -H "Content-Type: application/json" \ -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ - -X PUT ${ELASTICSEARCH_URL}/mock-threat-list/_doc/$i \ + -X PUT ${ELASTICSEARCH_URL}/mock-threat-list-1/_doc/$i \ --data " { \"@timestamp\": \"$(date -u +"%Y-%m-%dT%H:%M:%SZ")\", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/query_with_threat_mapping_perf.json b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/query_with_threat_mapping_perf.json new file mode 100644 index 00000000000000..c573db7fbca359 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/query_with_threat_mapping_perf.json @@ -0,0 +1,32 @@ +{ + "concurrent_searches": 10, + "items_per_search": 10, + "index": ["auditbeat-*", "endgame-*", "filebeat-*", "logs-*", "packetbeat-*", "winlogbeat-*"], + "name": "Indicator Match Concurrent Searches", + "description": "Does 100 Concurrent searches with 10 items per search", + "rule_id": "indicator_concurrent_search", + "risk_score": 1, + "severity": "high", + "type": "threat_match", + "query": "*:*", + "tags": ["concurrent_searches_test", "from_script"], + "threat_index": ["mock-threat-list-1"], + "threat_language": "kuery", + "threat_query": "*:*", + "threat_mapping": [ + { + "entries": [ + { + "field": "source.port", + "type": "mapping", + "value": "source.port" + }, + { + "field": "source.ip", + "type": "mapping", + "value": "source.ip" + } + ] + } + ] +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts index 4559a658c9583c..92e6b9562d9706 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts @@ -68,6 +68,8 @@ export const sampleRuleAlertParams = ( threat: undefined, version: 1, exceptionsList: getListArrayMock(), + concurrentSearches: undefined, + itemsPerSearch: undefined, }); export const sampleRuleSO = (): SavedObject => { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts index cfe71f66395b04..50e740e81830f8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts @@ -54,6 +54,8 @@ const signalSchema = schema.object({ threatQuery: schema.maybe(schema.string()), threatMapping: schema.maybe(schema.arrayOf(schema.object({}, { unknowns: 'allow' }))), threatLanguage: schema.maybe(schema.string()), + concurrentSearches: schema.maybe(schema.number()), + itemsPerSearch: schema.maybe(schema.number()), }); /** diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts index 415abc9d995fba..dc68e3949eb363 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts @@ -504,7 +504,7 @@ describe('rules_notification_alert_type', () => { await alert.executor(payload); expect(logger.error).toHaveBeenCalled(); expect(logger.error.mock.calls[0][0]).toContain( - 'An error occurred during rule execution: message: "Threat Match rule is missing threatQuery and/or threatIndex and/or threatMapping: threatQuery: "undefined" threatIndex: "undefined" threatMapping: "undefined"" name: "Detect Root/Admin Users" id: "04128c15-0d1b-4716-a4c5-46997ac7f3bd" rule id: "rule-1" signals index: ".siem-signals"' + 'An error occurred during rule execution: message: "Indicator match is missing threatQuery and/or threatIndex and/or threatMapping: threatQuery: "undefined" threatIndex: "undefined" threatMapping: "undefined"" name: "Detect Root/Admin Users" id: "04128c15-0d1b-4716-a4c5-46997ac7f3bd" rule id: "rule-1" signals index: ".siem-signals"' ); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index a0d5c833b208cb..1d2b1c23f868f5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -119,6 +119,8 @@ export const signalRulesAlertType = ({ timestampOverride, type, exceptionsList, + concurrentSearches, + itemsPerSearch, } = params; const searchAfterSize = Math.min(maxSignals, DEFAULT_SEARCH_AFTER_PAGE_SIZE); @@ -360,7 +362,7 @@ export const signalRulesAlertType = ({ ) { throw new Error( [ - 'Threat Match rule is missing threatQuery and/or threatIndex and/or threatMapping:', + 'Indicator match is missing threatQuery and/or threatIndex and/or threatMapping:', `threatQuery: "${threatQuery}"`, `threatIndex: "${threatIndex}"`, `threatMapping: "${threatMapping}"`, @@ -403,6 +405,8 @@ export const signalRulesAlertType = ({ threatLanguage, buildRuleMessage, threatIndex, + concurrentSearches: concurrentSearches ?? 1, + itemsPerSearch: itemsPerSearch ?? 9000, }); } else if (type === 'query' || type === 'saved_query') { const inputIndex = await getInputIndex(services, version, index); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.test.ts index 85d172b3631a93..8eed838fc9680b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.test.ts @@ -19,28 +19,32 @@ import { } from './build_threat_mapping_filter'; import { getThreatMappingMock, - getThreatListSearchResponseMock, getThreatListItemMock, getThreatMappingFilterMock, getFilterThreatMapping, getThreatMappingFiltersShouldMock, getThreatMappingFilterShouldMock, + getThreatListSearchResponseMock, } from './build_threat_mapping_filter.mock'; -import { BooleanFilter } from './types'; +import { BooleanFilter, ThreatListItem } from './types'; describe('build_threat_mapping_filter', () => { describe('buildThreatMappingFilter', () => { test('it should throw if given a chunk over 1024 in size', () => { const threatMapping = getThreatMappingMock(); - const threatList = getThreatListSearchResponseMock(); + const threatList = getThreatListSearchResponseMock().hits.hits; expect(() => - buildThreatMappingFilter({ threatMapping, threatList, chunkSize: 1025 }) + buildThreatMappingFilter({ + threatMapping, + threatList, + chunkSize: 1025, + }) ).toThrow('chunk sizes cannot exceed 1024 in size'); }); test('it should NOT throw if given a chunk under 1024 in size', () => { const threatMapping = getThreatMappingMock(); - const threatList = getThreatListSearchResponseMock(); + const threatList = getThreatListSearchResponseMock().hits.hits; expect(() => buildThreatMappingFilter({ threatMapping, threatList, chunkSize: 1023 }) ).not.toThrow(); @@ -48,30 +52,30 @@ describe('build_threat_mapping_filter', () => { test('it should create the correct entries when using the default mocks', () => { const threatMapping = getThreatMappingMock(); - const threatList = getThreatListSearchResponseMock(); + const threatList = getThreatListSearchResponseMock().hits.hits; const filter = buildThreatMappingFilter({ threatMapping, threatList }); expect(filter).toEqual(getThreatMappingFilterMock()); }); test('it should not mutate the original threatMapping', () => { const threatMapping = getThreatMappingMock(); - const threatList = getThreatListSearchResponseMock(); + const threatList = getThreatListSearchResponseMock().hits.hits; buildThreatMappingFilter({ threatMapping, threatList }); expect(threatMapping).toEqual(getThreatMappingMock()); }); test('it should not mutate the original threatListItem', () => { const threatMapping = getThreatMappingMock(); - const threatList = getThreatListSearchResponseMock(); + const threatList = getThreatListSearchResponseMock().hits.hits; buildThreatMappingFilter({ threatMapping, threatList }); - expect(threatList).toEqual(getThreatListSearchResponseMock()); + expect(threatList).toEqual(getThreatListSearchResponseMock().hits.hits); }); }); describe('filterThreatMapping', () => { test('it should not remove any entries when using the default mocks', () => { const threatMapping = getThreatMappingMock(); - const threatListItem = getThreatListItemMock(); + const threatListItem = getThreatListSearchResponseMock().hits.hits[0]; const item = filterThreatMapping({ threatMapping, threatListItem }); const expected = getFilterThreatMapping(); @@ -80,7 +84,7 @@ describe('build_threat_mapping_filter', () => { test('it should only give one filtered element if only 1 element is defined', () => { const [firstElement] = getThreatMappingMock(); // get only the first element - const threatListItem = getThreatListItemMock(); + const threatListItem = getThreatListSearchResponseMock().hits.hits[0]; const item = filterThreatMapping({ threatMapping: [firstElement], threatListItem }); const [firstElementFilter] = getFilterThreatMapping(); // get only the first element to compare @@ -89,7 +93,7 @@ describe('build_threat_mapping_filter', () => { test('it should not mutate the original threatMapping', () => { const threatMapping = getThreatMappingMock(); - const threatListItem = getThreatListItemMock(); + const threatListItem = getThreatListSearchResponseMock().hits.hits[0]; filterThreatMapping({ threatMapping, @@ -100,13 +104,13 @@ describe('build_threat_mapping_filter', () => { test('it should not mutate the original threatListItem', () => { const threatMapping = getThreatMappingMock(); - const threatListItem = getThreatListItemMock(); + const threatListItem = getThreatListSearchResponseMock().hits.hits[0]; filterThreatMapping({ threatMapping, threatListItem, }); - expect(threatListItem).toEqual(getThreatListItemMock()); + expect(threatListItem).toEqual(getThreatListSearchResponseMock().hits.hits[0]); }); test('it should remove the entire "AND" clause if one of the pieces of data is missing from the list', () => { @@ -166,9 +170,11 @@ describe('build_threat_mapping_filter', () => { }, ], threatListItem: { - '@timestamp': '2020-09-09T21:59:13Z', - host: { - name: 'host-1', + _source: { + '@timestamp': '2020-09-09T21:59:13Z', + host: { + name: 'host-1', + }, }, }, }); @@ -189,7 +195,7 @@ describe('build_threat_mapping_filter', () => { describe('createInnerAndClauses', () => { test('it should return two clauses given a single entry', () => { const [{ entries: threatMappingEntries }] = getThreatMappingMock(); // get the first element - const threatListItem = getThreatListItemMock(); + const threatListItem = getThreatListSearchResponseMock().hits.hits[0]; const innerClause = createInnerAndClauses({ threatMappingEntries, threatListItem }); const { bool: { @@ -219,7 +225,7 @@ describe('build_threat_mapping_filter', () => { type: 'mapping', }, ]; - const threatListItem = getThreatListItemMock(); + const threatListItem = getThreatListSearchResponseMock().hits.hits[0]; const innerClause = createInnerAndClauses({ threatMappingEntries, threatListItem }); const { bool: { @@ -248,7 +254,7 @@ describe('build_threat_mapping_filter', () => { type: 'mapping', }, ]; - const threatListItem = getThreatListItemMock(); + const threatListItem = getThreatListSearchResponseMock().hits.hits[0]; const innerClause = createInnerAndClauses({ threatMappingEntries, threatListItem }); const { bool: { @@ -275,7 +281,7 @@ describe('build_threat_mapping_filter', () => { type: 'mapping', }, ]; - const threatListItem = getThreatListItemMock(); + const threatListItem = getThreatListSearchResponseMock().hits.hits[0]; const innerClause = createInnerAndClauses({ threatMappingEntries, threatListItem }); expect(innerClause).toEqual([]); }); @@ -284,27 +290,31 @@ describe('build_threat_mapping_filter', () => { describe('createAndOrClauses', () => { test('it should return all clauses given the entries', () => { const threatMapping = getThreatMappingMock(); - const threatListItem = getThreatListItemMock(); + const threatListItem = getThreatListSearchResponseMock().hits.hits[0]; const innerClause = createAndOrClauses({ threatMapping, threatListItem }); expect(innerClause).toEqual(getThreatMappingFilterShouldMock()); }); test('it should filter out data from entries that do not have mappings', () => { const threatMapping = getThreatMappingMock(); - const threatListItem = { ...getThreatListItemMock(), foo: 'bar' }; + const threatListItem = getThreatListSearchResponseMock().hits.hits[0]; + threatListItem._source = { + ...getThreatListSearchResponseMock().hits.hits[0]._source, + foo: 'bar', + }; const innerClause = createAndOrClauses({ threatMapping, threatListItem }); expect(innerClause).toEqual(getThreatMappingFilterShouldMock()); }); test('it should return an empty boolean given an empty array', () => { - const threatListItem = getThreatListItemMock(); + const threatListItem = getThreatListSearchResponseMock().hits.hits[0]; const innerClause = createAndOrClauses({ threatMapping: [], threatListItem }); expect(innerClause).toEqual({ bool: { minimum_should_match: 1, should: [] } }); }); test('it should return an empty boolean clause given an empty object for a threat list item', () => { const threatMapping = getThreatMappingMock(); - const innerClause = createAndOrClauses({ threatMapping, threatListItem: {} }); + const innerClause = createAndOrClauses({ threatMapping, threatListItem: { _source: {} } }); expect(innerClause).toEqual({ bool: { minimum_should_match: 1, should: [] } }); }); }); @@ -312,7 +322,7 @@ describe('build_threat_mapping_filter', () => { describe('buildEntriesMappingFilter', () => { test('it should return all clauses given the entries', () => { const threatMapping = getThreatMappingMock(); - const threatList = getThreatListSearchResponseMock(); + const threatList = getThreatListSearchResponseMock().hits.hits; const mapping = buildEntriesMappingFilter({ threatMapping, threatList, @@ -326,8 +336,7 @@ describe('build_threat_mapping_filter', () => { test('it should return empty "should" given an empty threat list', () => { const threatMapping = getThreatMappingMock(); - const threatList = getThreatListSearchResponseMock(); - threatList.hits.hits = []; + const threatList: ThreatListItem[] = []; const mapping = buildEntriesMappingFilter({ threatMapping, threatList, @@ -340,7 +349,7 @@ describe('build_threat_mapping_filter', () => { }); test('it should return empty "should" given an empty threat mapping', () => { - const threatList = getThreatListSearchResponseMock(); + const threatList = getThreatListSearchResponseMock().hits.hits; const mapping = buildEntriesMappingFilter({ threatMapping: [], threatList, @@ -374,7 +383,7 @@ describe('build_threat_mapping_filter', () => { }, ], ]; - const threatList = getThreatListSearchResponseMock(); + const threatList = getThreatListSearchResponseMock().hits.hits; const mapping = buildEntriesMappingFilter({ threatMapping, threatList, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.ts index 346f156a9ec338..294d97e0bf2f12 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.ts @@ -53,9 +53,9 @@ export const filterThreatMapping = ({ }: FilterThreatMappingOptions): ThreatMapping => threatMapping .map((threatMap) => { - const atLeastOneItemMissingInThreatList = threatMap.entries.some( - (entry) => get(entry.value, threatListItem) == null - ); + const atLeastOneItemMissingInThreatList = threatMap.entries.some((entry) => { + return get(entry.value, threatListItem._source) == null; + }); if (atLeastOneItemMissingInThreatList) { return { ...threatMap, entries: [] }; } else { @@ -69,7 +69,7 @@ export const createInnerAndClauses = ({ threatListItem, }: CreateInnerAndClausesOptions): BooleanFilter[] => { return threatMappingEntries.reduce((accum, threatMappingEntry) => { - const value = get(threatMappingEntry.value, threatListItem); + const value = get(threatMappingEntry.value, threatListItem._source); if (value != null) { // These values could be potentially 10k+ large so mutating the array intentionally accum.push({ @@ -114,24 +114,21 @@ export const buildEntriesMappingFilter = ({ threatList, chunkSize, }: BuildEntriesMappingFilterOptions): BooleanFilter => { - const combinedShould = threatList.hits.hits.reduce( - (accum, threatListSearchItem) => { - const filteredEntries = filterThreatMapping({ - threatMapping, - threatListItem: threatListSearchItem._source, - }); - const queryWithAndOrClause = createAndOrClauses({ - threatMapping: filteredEntries, - threatListItem: threatListSearchItem._source, - }); - if (queryWithAndOrClause.bool.should.length !== 0) { - // These values can be 10k+ large, so using a push here for performance - accum.push(queryWithAndOrClause); - } - return accum; - }, - [] - ); + const combinedShould = threatList.reduce((accum, threatListSearchItem) => { + const filteredEntries = filterThreatMapping({ + threatMapping, + threatListItem: threatListSearchItem, + }); + const queryWithAndOrClause = createAndOrClauses({ + threatMapping: filteredEntries, + threatListItem: threatListSearchItem, + }); + if (queryWithAndOrClause.bool.should.length !== 0) { + // These values can be 10k+ large, so using a push here for performance + accum.push(queryWithAndOrClause); + } + return accum; + }, []); const should = splitShouldClauses({ should: combinedShould, chunkSize }); return { bool: { should, minimum_should_match: 1 } }; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts index 037f91240edfaa..43fb759d076203 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts @@ -4,13 +4,12 @@ * you may not use this file except in compliance with the Elastic License. */ -import { getThreatList } from './get_threat_list'; import { buildThreatMappingFilter } from './build_threat_mapping_filter'; import { getFilter } from '../get_filter'; import { searchAfterAndBulkCreate } from '../search_after_bulk_create'; -import { CreateThreatSignalOptions, ThreatSignalResults } from './types'; -import { combineResults } from './utils'; +import { CreateThreatSignalOptions } from './types'; +import { SearchAfterAndBulkCreateReturnType } from '../types'; export const createThreatSignal = async ({ threatMapping, @@ -41,28 +40,11 @@ export const createThreatSignal = async ({ refresh, tags, throttle, - threatFilters, - threatQuery, - threatLanguage, buildRuleMessage, - threatIndex, name, currentThreatList, currentResult, -}: CreateThreatSignalOptions): Promise => { - const threatList = await getThreatList({ - callCluster: services.callCluster, - exceptionItems, - query: threatQuery, - language: threatLanguage, - threatFilters, - index: threatIndex, - searchAfter: currentThreatList.hits.hits[currentThreatList.hits.hits.length - 1].sort, - sortField: undefined, - sortOrder: undefined, - listClient, - }); - +}: CreateThreatSignalOptions): Promise => { const threatFilter = buildThreatMappingFilter({ threatMapping, threatList: currentThreatList, @@ -71,7 +53,12 @@ export const createThreatSignal = async ({ if (threatFilter.query.bool.should.length === 0) { // empty threat list and we do not want to return everything as being // a hit so opt to return the existing result. - return { threatList, results: currentResult }; + logger.debug( + buildRuleMessage( + 'Indicator items are empty after filtering for missing data, returning without attempting a match' + ) + ); + return currentResult; } else { const esFilter = await getFilter({ type, @@ -83,7 +70,13 @@ export const createThreatSignal = async ({ index: inputIndex, lists: exceptionItems, }); - const newResult = await searchAfterAndBulkCreate({ + + logger.debug( + buildRuleMessage( + `${threatFilter.query.bool.should.length} indicator items are being checked for existence of matches` + ) + ); + const result = await searchAfterAndBulkCreate({ gap, previousStartedAt, listClient, @@ -110,7 +103,15 @@ export const createThreatSignal = async ({ throttle, buildRuleMessage, }); - const results = combineResults(currentResult, newResult); - return { threatList, results }; + logger.debug( + buildRuleMessage( + `${ + threatFilter.query.bool.should.length + } items have completed match checks and the total times to search were ${ + result.searchAfterTimes.length !== 0 ? result.searchAfterTimes : '(unknown) ' + }ms` + ) + ); + return result; } }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts index 8be76dc8caf0f9..e90c45d40de950 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts @@ -4,11 +4,13 @@ * you may not use this file except in compliance with the Elastic License. */ -import { getThreatList } from './get_threat_list'; +import chunk from 'lodash/fp/chunk'; +import { getThreatList, getThreatListCount } from './get_threat_list'; import { CreateThreatSignalsOptions } from './types'; import { createThreatSignal } from './create_threat_signal'; import { SearchAfterAndBulkCreateReturnType } from '../types'; +import { combineConcurrentResults } from './utils'; export const createThreatSignals = async ({ threatMapping, @@ -45,7 +47,12 @@ export const createThreatSignals = async ({ buildRuleMessage, threatIndex, name, + concurrentSearches, + itemsPerSearch, }: CreateThreatSignalsOptions): Promise => { + logger.debug(buildRuleMessage('Indicator matching rule starting')); + const perPage = concurrentSearches * itemsPerSearch; + let results: SearchAfterAndBulkCreateReturnType = { success: true, bulkCreateTimes: [], @@ -55,6 +62,16 @@ export const createThreatSignals = async ({ errors: [], }; + let threatListCount = await getThreatListCount({ + callCluster: services.callCluster, + exceptionItems, + threatFilters, + query: threatQuery, + language: threatLanguage, + index: threatIndex, + }); + logger.debug(buildRuleMessage(`Total indicator items: ${threatListCount}`)); + let threatList = await getThreatList({ callCluster: services.callCluster, exceptionItems, @@ -66,47 +83,89 @@ export const createThreatSignals = async ({ searchAfter: undefined, sortField: undefined, sortOrder: undefined, + logger, + buildRuleMessage, + perPage, }); - while (threatList.hits.hits.length !== 0 && results.createdSignalsCount <= params.maxSignals) { - ({ threatList, results } = await createThreatSignal({ - threatMapping, - query, - inputIndex, - type, - filters, - language, - savedId, - services, + while (threatList.hits.hits.length !== 0) { + const chunks = chunk(itemsPerSearch, threatList.hits.hits); + logger.debug(buildRuleMessage(`${chunks.length} concurrent indicator searches are starting.`)); + const concurrentSearchesPerformed = chunks.map>( + (slicedChunk) => + createThreatSignal({ + threatMapping, + query, + inputIndex, + type, + filters, + language, + savedId, + services, + exceptionItems, + gap, + previousStartedAt, + listClient, + logger, + eventsTelemetry, + alertId, + outputIndex, + params, + searchAfterSize, + actions, + createdBy, + createdAt, + updatedBy, + updatedAt, + interval, + enabled, + tags, + refresh, + throttle, + buildRuleMessage, + name, + currentThreatList: slicedChunk, + currentResult: results, + }) + ); + const searchesPerformed = await Promise.all(concurrentSearchesPerformed); + results = combineConcurrentResults(results, searchesPerformed); + threatListCount -= threatList.hits.hits.length; + logger.debug( + buildRuleMessage( + `Concurrent indicator match searches completed with ${results.createdSignalsCount} signals found`, + `search times of ${results.searchAfterTimes}ms,`, + `bulk create times ${results.bulkCreateTimes}ms,`, + `all successes are ${results.success}` + ) + ); + if (results.createdSignalsCount >= params.maxSignals) { + logger.debug( + buildRuleMessage( + `Indicator match has reached its max signals count ${params.maxSignals}. Additional indicator items not checked are ${threatListCount}` + ) + ); + break; + } + logger.debug(buildRuleMessage(`Indicator items left to check are ${threatListCount}`)); + + threatList = await getThreatList({ + callCluster: services.callCluster, exceptionItems, - gap, - previousStartedAt, - listClient, - logger, - eventsTelemetry, - alertId, - outputIndex, - params, - searchAfterSize, - actions, - createdBy, - createdAt, - updatedBy, - updatedAt, - interval, - enabled, - tags, - refresh, - throttle, + query: threatQuery, + language: threatLanguage, threatFilters, - threatQuery, + index: threatIndex, + searchAfter: threatList.hits.hits[threatList.hits.hits.length - 1].sort, + sortField: undefined, + sortOrder: undefined, + listClient, buildRuleMessage, - threatIndex, - threatLanguage, - name, - currentThreatList: threatList, - currentResult: results, - })); + logger, + perPage, + }); } + + logger.debug(buildRuleMessage('Indicator matching rule has completed')); return results; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_threat_list.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_threat_list.ts index 3147eb1705168a..aba3f6f69d706f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_threat_list.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/get_threat_list.ts @@ -10,6 +10,7 @@ import { GetSortWithTieBreakerOptions, GetThreatListOptions, SortWithTieBreaker, + ThreatListCountOptions, ThreatListItem, } from './types'; @@ -30,6 +31,8 @@ export const getThreatList = async ({ exceptionItems, threatFilters, listClient, + buildRuleMessage, + logger, }: GetThreatListOptions): Promise> => { const calculatedPerPage = perPage ?? MAX_PER_PAGE; if (calculatedPerPage > 10000) { @@ -43,6 +46,11 @@ export const getThreatList = async ({ exceptionItems ); + logger.debug( + buildRuleMessage( + `Querying the indicator items from the index: "${index}" with searchAfter: "${searchAfter}" for up to ${calculatedPerPage} indicator items` + ) + ); const response: SearchResponse = await callCluster('search', { body: { query: queryFilter, @@ -58,6 +66,8 @@ export const getThreatList = async ({ index, size: calculatedPerPage, }); + + logger.debug(buildRuleMessage(`Retrieved indicator items of size: ${response.hits.hits.length}`)); return response; }; @@ -89,3 +99,30 @@ export const getSortWithTieBreaker = ({ } } }; + +export const getThreatListCount = async ({ + callCluster, + query, + language, + threatFilters, + index, + exceptionItems, +}: ThreatListCountOptions): Promise => { + const queryFilter = getQueryFilter( + query, + language ?? 'kuery', + threatFilters, + index, + exceptionItems + ); + const response: { + count: number; + } = await callCluster('count', { + body: { + query: queryFilter, + }, + ignoreUnavailable: true, + index, + }); + return response.count; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts index 0078cf1b3c64f8..2e32a4e682403f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts @@ -5,7 +5,6 @@ */ import { Duration } from 'moment'; -import { SearchResponse } from 'elasticsearch'; import { ListClient } from '../../../../../../lists/server'; import { Type, @@ -17,6 +16,8 @@ import { ThreatMappingEntries, ThreatIndex, ThreatLanguageOrUndefined, + ConcurrentSearches, + ItemsPerSearch, } from '../../../../../common/detection_engine/schemas/types/threat_mapping'; import { PartialFilter, RuleTypeParams } from '../../types'; import { AlertServices } from '../../../../../../alerts/server'; @@ -62,6 +63,8 @@ export interface CreateThreatSignalsOptions { threatIndex: ThreatIndex; threatLanguage: ThreatLanguageOrUndefined; name: string; + concurrentSearches: ConcurrentSearches; + itemsPerSearch: ItemsPerSearch; } export interface CreateThreatSignalOptions { @@ -93,24 +96,15 @@ export interface CreateThreatSignalOptions { tags: string[]; refresh: false | 'wait_for'; throttle: string; - threatFilters: PartialFilter[]; - threatQuery: ThreatQuery; buildRuleMessage: BuildRuleMessage; - threatIndex: ThreatIndex; - threatLanguage: ThreatLanguageOrUndefined; name: string; - currentThreatList: SearchResponse; + currentThreatList: ThreatListItem[]; currentResult: SearchAfterAndBulkCreateReturnType; } -export interface ThreatSignalResults { - threatList: SearchResponse; - results: SearchAfterAndBulkCreateReturnType; -} - export interface BuildThreatMappingFilterOptions { threatMapping: ThreatMapping; - threatList: SearchResponse; + threatList: ThreatListItem[]; chunkSize?: number; } @@ -131,7 +125,7 @@ export interface CreateAndOrClausesOptions { export interface BuildEntriesMappingFilterOptions { threatMapping: ThreatMapping; - threatList: SearchResponse; + threatList: ThreatListItem[]; chunkSize: number; } @@ -156,6 +150,17 @@ export interface GetThreatListOptions { threatFilters: PartialFilter[]; exceptionItems: ExceptionListItemSchema[]; listClient: ListClient; + buildRuleMessage: BuildRuleMessage; + logger: Logger; +} + +export interface ThreatListCountOptions { + callCluster: ILegacyScopedClusterClient['callAsCurrentUser']; + query: string; + language: ThreatLanguageOrUndefined; + threatFilters: PartialFilter[]; + index: string[]; + exceptionItems: ExceptionListItemSchema[]; } export interface GetSortWithTieBreakerOptions { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.test.ts index 27593b40b0c8f6..840d64381c7932 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.test.ts @@ -6,7 +6,13 @@ import { SearchAfterAndBulkCreateReturnType } from '../types'; -import { calculateAdditiveMax, combineResults } from './utils'; +import { + calculateAdditiveMax, + calculateMax, + calculateMaxLookBack, + combineConcurrentResults, + combineResults, +} from './utils'; describe('utils', () => { describe('calculateAdditiveMax', () => { @@ -156,4 +162,383 @@ describe('utils', () => { ); }); }); + + describe('calculateMax', () => { + test('it should return 0 for two empty arrays', () => { + const max = calculateMax([], []); + expect(max).toEqual('0'); + }); + + test('it should return 5 for two arrays with the numbers 5', () => { + const max = calculateMax(['5'], ['5']); + expect(max).toEqual('5'); + }); + + test('it should return 5 for two arrays with second array having just 5', () => { + const max = calculateMax([], ['5']); + expect(max).toEqual('5'); + }); + + test('it should return 5 for two arrays with first array having just 5', () => { + const max = calculateMax(['5'], []); + expect(max).toEqual('5'); + }); + + test('it should return 10 for the max of the two arrays when the max of each array is 10', () => { + const max = calculateMax(['3', '5', '1'], ['3', '5', '10']); + expect(max).toEqual('10'); + }); + + test('it should return 10 for the max of the two arrays when the max of the first is 10', () => { + const max = calculateMax(['3', '5', '10'], ['3', '5', '1']); + expect(max).toEqual('10'); + }); + }); + + describe('calculateMaxLookBack', () => { + test('it should return null if both are null', () => { + const max = calculateMaxLookBack(null, null); + expect(max).toEqual(null); + }); + + test('it should return undefined if both are undefined', () => { + const max = calculateMaxLookBack(undefined, undefined); + expect(max).toEqual(undefined); + }); + + test('it should return null if both one is null and other other is undefined', () => { + const max = calculateMaxLookBack(undefined, null); + expect(max).toEqual(null); + }); + + test('it should return null if both one is null and other other is undefined with flipped arguments', () => { + const max = calculateMaxLookBack(null, undefined); + expect(max).toEqual(null); + }); + + test('it should return a date time if one argument is null', () => { + const max = calculateMaxLookBack(null, new Date('2020-09-16T03:34:32.390Z')); + expect(max).toEqual(new Date('2020-09-16T03:34:32.390Z')); + }); + + test('it should return a date time if one argument is null with flipped arguments', () => { + const max = calculateMaxLookBack(new Date('2020-09-16T03:34:32.390Z'), null); + expect(max).toEqual(new Date('2020-09-16T03:34:32.390Z')); + }); + + test('it should return a date time if one argument is undefined', () => { + const max = calculateMaxLookBack(new Date('2020-09-16T03:34:32.390Z'), undefined); + expect(max).toEqual(new Date('2020-09-16T03:34:32.390Z')); + }); + + test('it should return a date time if one argument is undefined with flipped arguments', () => { + const max = calculateMaxLookBack(undefined, new Date('2020-09-16T03:34:32.390Z')); + expect(max).toEqual(new Date('2020-09-16T03:34:32.390Z')); + }); + + test('it should return a date time that is larger than the other', () => { + const max = calculateMaxLookBack( + new Date('2020-10-16T03:34:32.390Z'), + new Date('2020-09-16T03:34:32.390Z') + ); + expect(max).toEqual(new Date('2020-10-16T03:34:32.390Z')); + }); + + test('it should return a date time that is larger than the other with arguments flipped', () => { + const max = calculateMaxLookBack( + new Date('2020-09-16T03:34:32.390Z'), + new Date('2020-10-16T03:34:32.390Z') + ); + expect(max).toEqual(new Date('2020-10-16T03:34:32.390Z')); + }); + }); + + describe('combineConcurrentResults', () => { + test('it should use the maximum found if given an empty array for newResults', () => { + const existingResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: [], + }; + const expectedResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['30'], // max value from existingResult.searchAfterTimes + bulkCreateTimes: ['25'], // max value from existingResult.bulkCreateTimes + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: [], + }; + const combinedResults = combineConcurrentResults(existingResult, []); + expect(combinedResults).toEqual(expectedResult); + }); + + test('it should work with empty arrays for searchAfterTimes and bulkCreateTimes', () => { + const existingResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: [], + }; + const newResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: [], + bulkCreateTimes: [], + lastLookBackDate: undefined, + createdSignalsCount: 0, + errors: [], + }; + const expectedResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['30'], // max value from existingResult.searchAfterTimes + bulkCreateTimes: ['25'], // max value from existingResult.bulkCreateTimes + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: [], + }; + + const combinedResults = combineConcurrentResults(existingResult, [newResult]); + expect(combinedResults).toEqual(expectedResult); + }); + + test('it should get the max of two new results and then combine the result with an existingResult correctly', () => { + const existingResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], // max is 30 + bulkCreateTimes: ['5', '15', '25'], // max is 25 + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: [], + }; + const newResult1: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: new Date('2020-09-16T03:34:32.390Z'), + createdSignalsCount: 5, + errors: [], + }; + const newResult2: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['40', '5', '15'], + bulkCreateTimes: ['50', '5', '15'], + lastLookBackDate: new Date('2020-09-16T04:34:32.390Z'), + createdSignalsCount: 8, + errors: [], + }; + + const expectedResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['70'], // max value between newResult1 and newResult2 + max array value of existingResult (40 + 30 = 70) + bulkCreateTimes: ['75'], // max value between newResult1 and newResult2 + max array value of existingResult (50 + 25 = 75) + lastLookBackDate: new Date('2020-09-16T04:34:32.390Z'), // max lastLookBackDate + createdSignalsCount: 16, // all the signals counted together (8 + 5 + 3) + errors: [], + }; + + const combinedResults = combineConcurrentResults(existingResult, [newResult1, newResult2]); + expect(combinedResults).toEqual(expectedResult); + }); + + test('it should get the max of two new results and then combine the result with an existingResult correctly when the results are flipped around', () => { + const existingResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], // max is 30 + bulkCreateTimes: ['5', '15', '25'], // max is 25 + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: [], + }; + const newResult1: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: new Date('2020-09-16T03:34:32.390Z'), + createdSignalsCount: 5, + errors: [], + }; + const newResult2: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['40', '5', '15'], + bulkCreateTimes: ['50', '5', '15'], + lastLookBackDate: new Date('2020-09-16T04:34:32.390Z'), + createdSignalsCount: 8, + errors: [], + }; + + const expectedResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['70'], // max value between newResult1 and newResult2 + max array value of existingResult (40 + 30 = 70) + bulkCreateTimes: ['75'], // max value between newResult1 and newResult2 + max array value of existingResult (50 + 25 = 75) + lastLookBackDate: new Date('2020-09-16T04:34:32.390Z'), // max lastLookBackDate + createdSignalsCount: 16, // all the signals counted together (8 + 5 + 3) + errors: [], + }; + + const combinedResults = combineConcurrentResults(existingResult, [newResult2, newResult1]); // two array elements are flipped + expect(combinedResults).toEqual(expectedResult); + }); + + test('it should return the max date correctly if one date contains a null', () => { + const existingResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], // max is 30 + bulkCreateTimes: ['5', '15', '25'], // max is 25 + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: [], + }; + const newResult1: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: new Date('2020-09-16T03:34:32.390Z'), + createdSignalsCount: 5, + errors: [], + }; + const newResult2: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['40', '5', '15'], + bulkCreateTimes: ['50', '5', '15'], + lastLookBackDate: null, + createdSignalsCount: 8, + errors: [], + }; + + const expectedResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['70'], // max value between newResult1 and newResult2 + max array value of existingResult (40 + 30 = 70) + bulkCreateTimes: ['75'], // max value between newResult1 and newResult2 + max array value of existingResult (50 + 25 = 75) + lastLookBackDate: new Date('2020-09-16T03:34:32.390Z'), // max lastLookBackDate + createdSignalsCount: 16, // all the signals counted together (8 + 5 + 3) + errors: [], + }; + + const combinedResults = combineConcurrentResults(existingResult, [newResult1, newResult2]); + expect(combinedResults).toEqual(expectedResult); + }); + + test('it should combine two results with success set to "true" if both are "true"', () => { + const existingResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: [], + }; + + const newResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: [], + }; + const combinedResults = combineConcurrentResults(existingResult, [newResult]); + expect(combinedResults.success).toEqual(true); + }); + + test('it should combine two results with success set to "false" if one of them is "false"', () => { + const existingResult: SearchAfterAndBulkCreateReturnType = { + success: false, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: [], + }; + + const newResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: [], + }; + const combinedResults = combineConcurrentResults(existingResult, [newResult]); + expect(combinedResults.success).toEqual(false); + }); + + test('it should use the latest date if it is set in the new result', () => { + const existingResult: SearchAfterAndBulkCreateReturnType = { + success: false, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: [], + }; + + const newResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: new Date('2020-09-16T03:34:32.390Z'), + createdSignalsCount: 3, + errors: [], + }; + const combinedResults = combineConcurrentResults(existingResult, [newResult]); + expect(combinedResults.lastLookBackDate?.toISOString()).toEqual('2020-09-16T03:34:32.390Z'); + }); + + test('it should combine the searchAfterTimes and the bulkCreateTimes', () => { + const existingResult: SearchAfterAndBulkCreateReturnType = { + success: false, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: [], + }; + + const newResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: new Date('2020-09-16T03:34:32.390Z'), + createdSignalsCount: 3, + errors: [], + }; + const combinedResults = combineConcurrentResults(existingResult, [newResult]); + expect(combinedResults).toEqual( + expect.objectContaining({ + searchAfterTimes: ['60'], + bulkCreateTimes: ['50'], + }) + ); + }); + + test('it should combine errors together without duplicates', () => { + const existingResult: SearchAfterAndBulkCreateReturnType = { + success: false, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: ['error 1', 'error 2', 'error 3'], + }; + + const newResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: new Date('2020-09-16T03:34:32.390Z'), + createdSignalsCount: 3, + errors: ['error 4', 'error 1', 'error 3', 'error 5'], + }; + const combinedResults = combineConcurrentResults(existingResult, [newResult]); + expect(combinedResults).toEqual( + expect.objectContaining({ + errors: ['error 1', 'error 2', 'error 3', 'error 4', 'error 5'], + }) + ); + }); + }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.ts index 401a4a1acb0652..d6c91fad6d9cb0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.ts @@ -19,6 +19,41 @@ export const calculateAdditiveMax = (existingTimers: string[], newTimers: string return [String(numericNewTimerMax + numericExistingTimerMax)]; }; +/** + * Given two timers this will take the max of each and then get the max from each. + * Max(Max(timer_array_1), Max(timer_array_2)) + * @param existingTimers String array of existing timers + * @param newTimers String array of new timers. + * @returns String array of the new maximum between the two timers + */ +export const calculateMax = (existingTimers: string[], newTimers: string[]): string => { + const numericNewTimerMax = Math.max(0, ...newTimers.map((time) => +time)); + const numericExistingTimerMax = Math.max(0, ...existingTimers.map((time) => +time)); + return String(Math.max(numericNewTimerMax, numericExistingTimerMax)); +}; + +/** + * Given two dates this will return the larger of the two unless one of them is null + * or undefined. If both one or the other is null/undefined it will return the newDate. + * If there is a mix of "undefined" and "null", this will prefer to set it to "null" as having + * a higher value than "undefined" + * @param existingDate The existing date which can be undefined or null or a date + * @param newDate The new date which can be undefined or null or a date + */ +export const calculateMaxLookBack = ( + existingDate: Date | null | undefined, + newDate: Date | null | undefined +): Date | null | undefined => { + const newDateValue = newDate === null ? 1 : newDate === undefined ? 0 : newDate.valueOf(); + const existingDateValue = + existingDate === null ? 1 : existingDate === undefined ? 0 : existingDate.valueOf(); + if (newDateValue >= existingDateValue) { + return newDate; + } else { + return existingDate; + } +}; + /** * Combines two results together and returns the results combined * @param currentResult The current result to combine with a newResult @@ -38,3 +73,39 @@ export const combineResults = ( createdSignalsCount: currentResult.createdSignalsCount + newResult.createdSignalsCount, errors: [...new Set([...currentResult.errors, ...newResult.errors])], }); + +/** + * Combines two results together and returns the results combined + * @param currentResult The current result to combine with a newResult + * @param newResult The new result to combine + */ +export const combineConcurrentResults = ( + currentResult: SearchAfterAndBulkCreateReturnType, + newResult: SearchAfterAndBulkCreateReturnType[] +): SearchAfterAndBulkCreateReturnType => { + const maxedNewResult = newResult.reduce( + (accum, item) => { + const maxSearchAfterTime = calculateMax(accum.searchAfterTimes, item.searchAfterTimes); + const maxBulkCreateTimes = calculateMax(accum.bulkCreateTimes, item.bulkCreateTimes); + const lastLookBackDate = calculateMaxLookBack(accum.lastLookBackDate, item.lastLookBackDate); + return { + success: accum.success && item.success, + searchAfterTimes: [maxSearchAfterTime], + bulkCreateTimes: [maxBulkCreateTimes], + lastLookBackDate, + createdSignalsCount: accum.createdSignalsCount + item.createdSignalsCount, + errors: [...new Set([...accum.errors, ...item.errors])], + }; + }, + { + success: true, + searchAfterTimes: [], + bulkCreateTimes: [], + lastLookBackDate: undefined, + createdSignalsCount: 0, + errors: [], + } + ); + + return combineResults(currentResult, maxedNewResult); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/types.ts index cf4d989c1f4c84..5cac76e2b0c014 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/types.ts @@ -44,6 +44,8 @@ import { ThreatQueryOrUndefined, ThreatMappingOrUndefined, ThreatLanguageOrUndefined, + ConcurrentSearchesOrUndefined, + ItemsPerSearchOrUndefined, } from '../../../common/detection_engine/schemas/types/threat_mapping'; import { LegacyCallAPIOptions } from '../../../../../../src/core/server'; @@ -93,6 +95,8 @@ export interface RuleTypeParams { references: References; version: Version; exceptionsList: ListArrayOrUndefined; + concurrentSearches: ConcurrentSearchesOrUndefined; + itemsPerSearch: ItemsPerSearchOrUndefined; } // eslint-disable-next-line @typescript-eslint/no-explicit-any