diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/create_threat_data.sh b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/create_threat_data.sh index 23c1914387c44d..4807afd71e8d2c 100755 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/create_threat_data.sh +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/create_threat_data.sh @@ -12,7 +12,7 @@ set -e # Adds port mock data to a threat list for testing. # Example: ./create_threat_data.sh -# Example: ./create_threat_data.sh 1000 2000 +# Example: ./create_threat_data.sh 1 500 START=${1:-1} END=${2:-1000} @@ -22,7 +22,7 @@ do { curl -s -k \ -H "Content-Type: application/json" \ -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ - -X PUT ${ELASTICSEARCH_URL}/mock-threat-list/_doc/$i \ + -X PUT ${ELASTICSEARCH_URL}/mock-threat-list-1/_doc/$i \ --data " { \"@timestamp\": \"$(date -u +"%Y-%m-%dT%H:%M:%SZ")\", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/query_with_threat_mapping_perf.json b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/query_with_threat_mapping_perf.json index 5a166ef32634e5..c573db7fbca359 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/query_with_threat_mapping_perf.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/rules/queries/query_with_threat_mapping_perf.json @@ -1,15 +1,17 @@ { - "concurrent_searches": 100, + "concurrent_searches": 10, "items_per_search": 10, - "name": "Threat Indicator Concurrent Searches", + "index": ["auditbeat-*", "endgame-*", "filebeat-*", "logs-*", "packetbeat-*", "winlogbeat-*"], + "name": "Indicator Match Concurrent Searches", "description": "Does 100 Concurrent searches with 10 items per search", - "rule_id": "threat_mapping_concurrent_search", + "rule_id": "indicator_concurrent_search", "risk_score": 1, "severity": "high", "type": "threat_match", "query": "*:*", "tags": ["concurrent_searches_test", "from_script"], - "threat_index": ["mock-threat-list"], + "threat_index": ["mock-threat-list-1"], + "threat_language": "kuery", "threat_query": "*:*", "threat_mapping": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts index 415abc9d995fba..dc68e3949eb363 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts @@ -504,7 +504,7 @@ describe('rules_notification_alert_type', () => { await alert.executor(payload); expect(logger.error).toHaveBeenCalled(); expect(logger.error.mock.calls[0][0]).toContain( - 'An error occurred during rule execution: message: "Threat Match rule is missing threatQuery and/or threatIndex and/or threatMapping: threatQuery: "undefined" threatIndex: "undefined" threatMapping: "undefined"" name: "Detect Root/Admin Users" id: "04128c15-0d1b-4716-a4c5-46997ac7f3bd" rule id: "rule-1" signals index: ".siem-signals"' + 'An error occurred during rule execution: message: "Indicator match is missing threatQuery and/or threatIndex and/or threatMapping: threatQuery: "undefined" threatIndex: "undefined" threatMapping: "undefined"" name: "Detect Root/Admin Users" id: "04128c15-0d1b-4716-a4c5-46997ac7f3bd" rule id: "rule-1" signals index: ".siem-signals"' ); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts index 5f79e50232ee59..43fb759d076203 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts @@ -9,7 +9,6 @@ import { buildThreatMappingFilter } from './build_threat_mapping_filter'; import { getFilter } from '../get_filter'; import { searchAfterAndBulkCreate } from '../search_after_bulk_create'; import { CreateThreatSignalOptions } from './types'; -import { combineResults } from './utils'; import { SearchAfterAndBulkCreateReturnType } from '../types'; export const createThreatSignal = async ({ @@ -77,7 +76,7 @@ export const createThreatSignal = async ({ `${threatFilter.query.bool.should.length} indicator items are being checked for existence of matches` ) ); - const newResult = await searchAfterAndBulkCreate({ + const result = await searchAfterAndBulkCreate({ gap, previousStartedAt, listClient, @@ -104,16 +103,15 @@ export const createThreatSignal = async ({ throttle, buildRuleMessage, }); - const results = combineResults(currentResult, newResult); logger.debug( buildRuleMessage( `${ threatFilter.query.bool.should.length - } items have completed match checks and the total time to search was ${ - newResult.searchAfterTimes.length !== 0 ? newResult.searchAfterTimes : '(unknown) ' + } items have completed match checks and the total times to search were ${ + result.searchAfterTimes.length !== 0 ? result.searchAfterTimes : '(unknown) ' }ms` ) ); - return results; + return result; } }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts index ae43ab4edce293..08292dacbc1870 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts @@ -50,7 +50,7 @@ export const createThreatSignals = async ({ concurrentSearches, itemsPerSearch, }: CreateThreatSignalsOptions): Promise => { - logger.debug(buildRuleMessage('Indicator matching starting')); + logger.debug(buildRuleMessage('Indicator matching rule starting')); const perPage = concurrentSearches * itemsPerSearch; let results: SearchAfterAndBulkCreateReturnType = { @@ -70,7 +70,7 @@ export const createThreatSignals = async ({ language: threatLanguage, index: threatIndex, }); - logger.debug(buildRuleMessage(`Total indicator items are ${threatListCount}`)); + logger.debug(buildRuleMessage(`Total indicator items: ${threatListCount}`)); let threatList = await getThreatList({ callCluster: services.callCluster, @@ -88,14 +88,13 @@ export const createThreatSignals = async ({ perPage, }); - const chunks = chunk(itemsPerSearch, threatList.hits.hits); - logger.debug( - buildRuleMessage( - `${chunks.length} concurrent indicator searches are starting. Each search has ${itemsPerSearch} indicator items` - ) - ); - while (threatList.hits.hits.length !== 0) { + const chunks = chunk(itemsPerSearch, threatList.hits.hits); + logger.debug( + buildRuleMessage( + `${chunks.length} concurrent indicator searches are starting. Each search can have up to ${itemsPerSearch} indicator items per search` + ) + ); const concurrentSearchesPerformed = chunks.map>( (slicedChunk) => createThreatSignal({ @@ -171,6 +170,6 @@ export const createThreatSignals = async ({ }); } - logger.debug(buildRuleMessage('Indicator Matching completed')); + logger.debug(buildRuleMessage('Indicator matching rule has completed')); return results; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.test.ts index 27593b40b0c8f6..840d64381c7932 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.test.ts @@ -6,7 +6,13 @@ import { SearchAfterAndBulkCreateReturnType } from '../types'; -import { calculateAdditiveMax, combineResults } from './utils'; +import { + calculateAdditiveMax, + calculateMax, + calculateMaxLookBack, + combineConcurrentResults, + combineResults, +} from './utils'; describe('utils', () => { describe('calculateAdditiveMax', () => { @@ -156,4 +162,383 @@ describe('utils', () => { ); }); }); + + describe('calculateMax', () => { + test('it should return 0 for two empty arrays', () => { + const max = calculateMax([], []); + expect(max).toEqual('0'); + }); + + test('it should return 5 for two arrays with the numbers 5', () => { + const max = calculateMax(['5'], ['5']); + expect(max).toEqual('5'); + }); + + test('it should return 5 for two arrays with second array having just 5', () => { + const max = calculateMax([], ['5']); + expect(max).toEqual('5'); + }); + + test('it should return 5 for two arrays with first array having just 5', () => { + const max = calculateMax(['5'], []); + expect(max).toEqual('5'); + }); + + test('it should return 10 for the max of the two arrays when the max of each array is 10', () => { + const max = calculateMax(['3', '5', '1'], ['3', '5', '10']); + expect(max).toEqual('10'); + }); + + test('it should return 10 for the max of the two arrays when the max of the first is 10', () => { + const max = calculateMax(['3', '5', '10'], ['3', '5', '1']); + expect(max).toEqual('10'); + }); + }); + + describe('calculateMaxLookBack', () => { + test('it should return null if both are null', () => { + const max = calculateMaxLookBack(null, null); + expect(max).toEqual(null); + }); + + test('it should return undefined if both are undefined', () => { + const max = calculateMaxLookBack(undefined, undefined); + expect(max).toEqual(undefined); + }); + + test('it should return null if both one is null and other other is undefined', () => { + const max = calculateMaxLookBack(undefined, null); + expect(max).toEqual(null); + }); + + test('it should return null if both one is null and other other is undefined with flipped arguments', () => { + const max = calculateMaxLookBack(null, undefined); + expect(max).toEqual(null); + }); + + test('it should return a date time if one argument is null', () => { + const max = calculateMaxLookBack(null, new Date('2020-09-16T03:34:32.390Z')); + expect(max).toEqual(new Date('2020-09-16T03:34:32.390Z')); + }); + + test('it should return a date time if one argument is null with flipped arguments', () => { + const max = calculateMaxLookBack(new Date('2020-09-16T03:34:32.390Z'), null); + expect(max).toEqual(new Date('2020-09-16T03:34:32.390Z')); + }); + + test('it should return a date time if one argument is undefined', () => { + const max = calculateMaxLookBack(new Date('2020-09-16T03:34:32.390Z'), undefined); + expect(max).toEqual(new Date('2020-09-16T03:34:32.390Z')); + }); + + test('it should return a date time if one argument is undefined with flipped arguments', () => { + const max = calculateMaxLookBack(undefined, new Date('2020-09-16T03:34:32.390Z')); + expect(max).toEqual(new Date('2020-09-16T03:34:32.390Z')); + }); + + test('it should return a date time that is larger than the other', () => { + const max = calculateMaxLookBack( + new Date('2020-10-16T03:34:32.390Z'), + new Date('2020-09-16T03:34:32.390Z') + ); + expect(max).toEqual(new Date('2020-10-16T03:34:32.390Z')); + }); + + test('it should return a date time that is larger than the other with arguments flipped', () => { + const max = calculateMaxLookBack( + new Date('2020-09-16T03:34:32.390Z'), + new Date('2020-10-16T03:34:32.390Z') + ); + expect(max).toEqual(new Date('2020-10-16T03:34:32.390Z')); + }); + }); + + describe('combineConcurrentResults', () => { + test('it should use the maximum found if given an empty array for newResults', () => { + const existingResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: [], + }; + const expectedResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['30'], // max value from existingResult.searchAfterTimes + bulkCreateTimes: ['25'], // max value from existingResult.bulkCreateTimes + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: [], + }; + const combinedResults = combineConcurrentResults(existingResult, []); + expect(combinedResults).toEqual(expectedResult); + }); + + test('it should work with empty arrays for searchAfterTimes and bulkCreateTimes', () => { + const existingResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: [], + }; + const newResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: [], + bulkCreateTimes: [], + lastLookBackDate: undefined, + createdSignalsCount: 0, + errors: [], + }; + const expectedResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['30'], // max value from existingResult.searchAfterTimes + bulkCreateTimes: ['25'], // max value from existingResult.bulkCreateTimes + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: [], + }; + + const combinedResults = combineConcurrentResults(existingResult, [newResult]); + expect(combinedResults).toEqual(expectedResult); + }); + + test('it should get the max of two new results and then combine the result with an existingResult correctly', () => { + const existingResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], // max is 30 + bulkCreateTimes: ['5', '15', '25'], // max is 25 + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: [], + }; + const newResult1: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: new Date('2020-09-16T03:34:32.390Z'), + createdSignalsCount: 5, + errors: [], + }; + const newResult2: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['40', '5', '15'], + bulkCreateTimes: ['50', '5', '15'], + lastLookBackDate: new Date('2020-09-16T04:34:32.390Z'), + createdSignalsCount: 8, + errors: [], + }; + + const expectedResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['70'], // max value between newResult1 and newResult2 + max array value of existingResult (40 + 30 = 70) + bulkCreateTimes: ['75'], // max value between newResult1 and newResult2 + max array value of existingResult (50 + 25 = 75) + lastLookBackDate: new Date('2020-09-16T04:34:32.390Z'), // max lastLookBackDate + createdSignalsCount: 16, // all the signals counted together (8 + 5 + 3) + errors: [], + }; + + const combinedResults = combineConcurrentResults(existingResult, [newResult1, newResult2]); + expect(combinedResults).toEqual(expectedResult); + }); + + test('it should get the max of two new results and then combine the result with an existingResult correctly when the results are flipped around', () => { + const existingResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], // max is 30 + bulkCreateTimes: ['5', '15', '25'], // max is 25 + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: [], + }; + const newResult1: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: new Date('2020-09-16T03:34:32.390Z'), + createdSignalsCount: 5, + errors: [], + }; + const newResult2: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['40', '5', '15'], + bulkCreateTimes: ['50', '5', '15'], + lastLookBackDate: new Date('2020-09-16T04:34:32.390Z'), + createdSignalsCount: 8, + errors: [], + }; + + const expectedResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['70'], // max value between newResult1 and newResult2 + max array value of existingResult (40 + 30 = 70) + bulkCreateTimes: ['75'], // max value between newResult1 and newResult2 + max array value of existingResult (50 + 25 = 75) + lastLookBackDate: new Date('2020-09-16T04:34:32.390Z'), // max lastLookBackDate + createdSignalsCount: 16, // all the signals counted together (8 + 5 + 3) + errors: [], + }; + + const combinedResults = combineConcurrentResults(existingResult, [newResult2, newResult1]); // two array elements are flipped + expect(combinedResults).toEqual(expectedResult); + }); + + test('it should return the max date correctly if one date contains a null', () => { + const existingResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], // max is 30 + bulkCreateTimes: ['5', '15', '25'], // max is 25 + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: [], + }; + const newResult1: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: new Date('2020-09-16T03:34:32.390Z'), + createdSignalsCount: 5, + errors: [], + }; + const newResult2: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['40', '5', '15'], + bulkCreateTimes: ['50', '5', '15'], + lastLookBackDate: null, + createdSignalsCount: 8, + errors: [], + }; + + const expectedResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['70'], // max value between newResult1 and newResult2 + max array value of existingResult (40 + 30 = 70) + bulkCreateTimes: ['75'], // max value between newResult1 and newResult2 + max array value of existingResult (50 + 25 = 75) + lastLookBackDate: new Date('2020-09-16T03:34:32.390Z'), // max lastLookBackDate + createdSignalsCount: 16, // all the signals counted together (8 + 5 + 3) + errors: [], + }; + + const combinedResults = combineConcurrentResults(existingResult, [newResult1, newResult2]); + expect(combinedResults).toEqual(expectedResult); + }); + + test('it should combine two results with success set to "true" if both are "true"', () => { + const existingResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: [], + }; + + const newResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: [], + }; + const combinedResults = combineConcurrentResults(existingResult, [newResult]); + expect(combinedResults.success).toEqual(true); + }); + + test('it should combine two results with success set to "false" if one of them is "false"', () => { + const existingResult: SearchAfterAndBulkCreateReturnType = { + success: false, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: [], + }; + + const newResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: [], + }; + const combinedResults = combineConcurrentResults(existingResult, [newResult]); + expect(combinedResults.success).toEqual(false); + }); + + test('it should use the latest date if it is set in the new result', () => { + const existingResult: SearchAfterAndBulkCreateReturnType = { + success: false, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: [], + }; + + const newResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: new Date('2020-09-16T03:34:32.390Z'), + createdSignalsCount: 3, + errors: [], + }; + const combinedResults = combineConcurrentResults(existingResult, [newResult]); + expect(combinedResults.lastLookBackDate?.toISOString()).toEqual('2020-09-16T03:34:32.390Z'); + }); + + test('it should combine the searchAfterTimes and the bulkCreateTimes', () => { + const existingResult: SearchAfterAndBulkCreateReturnType = { + success: false, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: [], + }; + + const newResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: new Date('2020-09-16T03:34:32.390Z'), + createdSignalsCount: 3, + errors: [], + }; + const combinedResults = combineConcurrentResults(existingResult, [newResult]); + expect(combinedResults).toEqual( + expect.objectContaining({ + searchAfterTimes: ['60'], + bulkCreateTimes: ['50'], + }) + ); + }); + + test('it should combine errors together without duplicates', () => { + const existingResult: SearchAfterAndBulkCreateReturnType = { + success: false, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: undefined, + createdSignalsCount: 3, + errors: ['error 1', 'error 2', 'error 3'], + }; + + const newResult: SearchAfterAndBulkCreateReturnType = { + success: true, + searchAfterTimes: ['10', '20', '30'], + bulkCreateTimes: ['5', '15', '25'], + lastLookBackDate: new Date('2020-09-16T03:34:32.390Z'), + createdSignalsCount: 3, + errors: ['error 4', 'error 1', 'error 3', 'error 5'], + }; + const combinedResults = combineConcurrentResults(existingResult, [newResult]); + expect(combinedResults).toEqual( + expect.objectContaining({ + errors: ['error 1', 'error 2', 'error 3', 'error 4', 'error 5'], + }) + ); + }); + }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.ts index 23eaf333f468bd..d6c91fad6d9cb0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.ts @@ -19,6 +19,41 @@ export const calculateAdditiveMax = (existingTimers: string[], newTimers: string return [String(numericNewTimerMax + numericExistingTimerMax)]; }; +/** + * Given two timers this will take the max of each and then get the max from each. + * Max(Max(timer_array_1), Max(timer_array_2)) + * @param existingTimers String array of existing timers + * @param newTimers String array of new timers. + * @returns String array of the new maximum between the two timers + */ +export const calculateMax = (existingTimers: string[], newTimers: string[]): string => { + const numericNewTimerMax = Math.max(0, ...newTimers.map((time) => +time)); + const numericExistingTimerMax = Math.max(0, ...existingTimers.map((time) => +time)); + return String(Math.max(numericNewTimerMax, numericExistingTimerMax)); +}; + +/** + * Given two dates this will return the larger of the two unless one of them is null + * or undefined. If both one or the other is null/undefined it will return the newDate. + * If there is a mix of "undefined" and "null", this will prefer to set it to "null" as having + * a higher value than "undefined" + * @param existingDate The existing date which can be undefined or null or a date + * @param newDate The new date which can be undefined or null or a date + */ +export const calculateMaxLookBack = ( + existingDate: Date | null | undefined, + newDate: Date | null | undefined +): Date | null | undefined => { + const newDateValue = newDate === null ? 1 : newDate === undefined ? 0 : newDate.valueOf(); + const existingDateValue = + existingDate === null ? 1 : existingDate === undefined ? 0 : existingDate.valueOf(); + if (newDateValue >= existingDateValue) { + return newDate; + } else { + return existingDate; + } +}; + /** * Combines two results together and returns the results combined * @param currentResult The current result to combine with a newResult @@ -48,29 +83,29 @@ export const combineConcurrentResults = ( currentResult: SearchAfterAndBulkCreateReturnType, newResult: SearchAfterAndBulkCreateReturnType[] ): SearchAfterAndBulkCreateReturnType => { - const maxedNewResult = newResult.reduce((concurrentResults, concurrentItems) => { - const maxSearchAfterTimes = concurrentItems.searchAfterTimes.reduce((accum, item) => - String(Math.max(+accum, +item)) - ); - const maxBulkCreateTimes = concurrentItems.bulkCreateTimes.reduce((accum, item) => - String(Math.max(+accum, +item)) - ); - const lastLookBackDate = - concurrentItems.lastLookBackDate != null && - concurrentResults.lastLookBackDate != null && - concurrentItems.lastLookBackDate > concurrentResults.lastLookBackDate - ? concurrentItems.lastLookBackDate - : concurrentResults.lastLookBackDate; - return { - success: concurrentResults.success && concurrentItems.success, - searchAfterTimes: [maxSearchAfterTimes], - bulkCreateTimes: [maxBulkCreateTimes], - lastLookBackDate, - createdSignalsCount: - concurrentResults.createdSignalsCount + concurrentItems.createdSignalsCount, - errors: [...new Set([...concurrentResults.errors, ...concurrentItems.errors])], - }; - }); + const maxedNewResult = newResult.reduce( + (accum, item) => { + const maxSearchAfterTime = calculateMax(accum.searchAfterTimes, item.searchAfterTimes); + const maxBulkCreateTimes = calculateMax(accum.bulkCreateTimes, item.bulkCreateTimes); + const lastLookBackDate = calculateMaxLookBack(accum.lastLookBackDate, item.lastLookBackDate); + return { + success: accum.success && item.success, + searchAfterTimes: [maxSearchAfterTime], + bulkCreateTimes: [maxBulkCreateTimes], + lastLookBackDate, + createdSignalsCount: accum.createdSignalsCount + item.createdSignalsCount, + errors: [...new Set([...accum.errors, ...item.errors])], + }; + }, + { + success: true, + searchAfterTimes: [], + bulkCreateTimes: [], + lastLookBackDate: undefined, + createdSignalsCount: 0, + errors: [], + } + ); return combineResults(currentResult, maxedNewResult); };