From bfa70cca174ddc58becce59532d68ddc488df3c1 Mon Sep 17 00:00:00 2001 From: Alex Kahan Date: Thu, 9 Jul 2020 15:01:13 -0400 Subject: [PATCH 1/2] Deduplication of entries and items before sending to endpoint --- .../endpoint/lib/artifacts/lists.test.ts | 139 +++++++++++++++++- .../server/endpoint/lib/artifacts/lists.ts | 19 ++- 2 files changed, 153 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts index acde455f77cb4b..1a19306b2fd606 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts @@ -139,6 +139,139 @@ describe('buildEventTypeSignal', () => { }); }); + test('it should deduplicate exception entries', async () => { + const testEntries: EntriesArray = [ + { field: 'server.domain.text', operator: 'included', type: 'match', value: 'DOMAIN' }, + { field: 'server.domain.text', operator: 'included', type: 'match', value: 'DOMAIN' }, + { field: 'server.domain.text', operator: 'included', type: 'match', value: 'DOMAIN' }, + { field: 'server.ip', operator: 'included', type: 'match', value: '192.168.1.1' }, + { + field: 'host.hostname.text', + operator: 'included', + type: 'match_any', + value: ['estc', 'kibana'], + }, + ]; + + const expectedEndpointExceptions = { + type: 'simple', + entries: [ + { + field: 'server.domain', + operator: 'included', + type: 'exact_caseless', + value: 'DOMAIN', + }, + { + field: 'server.ip', + operator: 'included', + type: 'exact_cased', + value: '192.168.1.1', + }, + { + field: 'host.hostname', + operator: 'included', + type: 'exact_caseless_any', + value: ['estc', 'kibana'], + }, + ], + }; + + const first = getFoundExceptionListItemSchemaMock(); + first.data[0].entries = testEntries; + mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first); + + const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', 'v1'); + expect(resp).toEqual({ + entries: [expectedEndpointExceptions], + }); + }); + + test('it should not deduplicate exception entries across nested boundaries', async () => { + const testEntries: EntriesArray = [ + { + entries: [ + { field: 'nested.field', operator: 'included', type: 'match', value: 'some value' }, + ], + field: 'some.parentField', + type: 'nested', + }, + // Same as above but not inside the nest + { field: 'nested.field', operator: 'included', type: 'match', value: 'some value' }, + ]; + + const expectedEndpointExceptions = { + type: 'simple', + entries: [ + { + entries: [ + { + field: 'nested.field', + operator: 'included', + type: 'exact_cased', + value: 'some value', + }, + ], + field: 'some.parentField', + type: 'nested', + }, + { + field: 'nested.field', + operator: 'included', + type: 'exact_cased', + value: 'some value', + }, + ], + }; + + const first = getFoundExceptionListItemSchemaMock(); + first.data[0].entries = testEntries; + mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first); + + const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', 'v1'); + expect(resp).toEqual({ + entries: [expectedEndpointExceptions], + }); + }); + + test('it should deduplicate exception items', async () => { + const testEntries: EntriesArray = [ + { field: 'server.domain.text', operator: 'included', type: 'match', value: 'DOMAIN' }, + { field: 'server.ip', operator: 'included', type: 'match', value: '192.168.1.1' }, + ]; + + const expectedEndpointExceptions = { + type: 'simple', + entries: [ + { + field: 'server.domain', + operator: 'included', + type: 'exact_caseless', + value: 'DOMAIN', + }, + { + field: 'server.ip', + operator: 'included', + type: 'exact_cased', + value: '192.168.1.1', + }, + ], + }; + + const first = getFoundExceptionListItemSchemaMock(); + first.data[0].entries = testEntries; + + // Create a second exception item with the same entries + first.data[1] = getExceptionListItemSchemaMock(); + first.data[1].entries = testEntries; + mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first); + + const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', 'v1'); + expect(resp).toEqual({ + entries: [expectedEndpointExceptions], + }); + }); + test('it should ignore unsupported entries', async () => { // Lists and exists are not supported by the Endpoint const testEntries: EntriesArray = [ @@ -178,8 +311,9 @@ describe('buildEventTypeSignal', () => { }); test('it should convert the exception lists response to the proper endpoint format while paging', async () => { - // The first call returns one exception + // The first call returns two exceptions const first = getFoundExceptionListItemSchemaMock(); + first.data.push(getExceptionListItemSchemaMock()); // The second call returns two exceptions const second = getFoundExceptionListItemSchemaMock(); @@ -194,7 +328,8 @@ describe('buildEventTypeSignal', () => { .mockReturnValueOnce(second) .mockReturnValueOnce(third); const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', 'v1'); - expect(resp.entries.length).toEqual(3); + // Expect 2 exceptions, the first two calls returned the same exception list items + expect(resp.entries.length).toEqual(2); }); test('it should handle no exceptions', async () => { diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts index 556405adff62f9..e08bd3f135d972 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts @@ -97,10 +97,18 @@ export function translateToEndpointExceptions( exc: FoundExceptionListItemSchema, schemaVersion: string ): TranslatedExceptionListItem[] { + const entrySet = new Set(); + const entriesFiltered: TranslatedExceptionListItem[] = []; if (schemaVersion === 'v1') { - return exc.data.map((item) => { - return translateItem(schemaVersion, item); + exc.data.forEach((item) => { + const translatedItem = translateItem(schemaVersion, item); + const itemHash = createHash('sha256').update(JSON.stringify(translatedItem)).digest('hex'); + if (!entrySet.has(itemHash)) { + entriesFiltered.push(translatedItem); + entrySet.add(itemHash); + } }); + return entriesFiltered; } else { throw new Error('unsupported schemaVersion'); } @@ -124,12 +132,17 @@ function translateItem( schemaVersion: string, item: ExceptionListItemSchema ): TranslatedExceptionListItem { + const itemSet = new Set(); return { type: item.type, entries: item.entries.reduce((translatedEntries: TranslatedEntry[], entry) => { const translatedEntry = translateEntry(schemaVersion, entry); if (translatedEntry !== undefined && translatedEntryType.is(translatedEntry)) { - translatedEntries.push(translatedEntry); + const itemHash = createHash('sha256').update(JSON.stringify(translatedEntry)).digest('hex'); + if (!itemSet.has(itemHash)) { + translatedEntries.push(translatedEntry); + itemSet.add(itemHash); + } } return translatedEntries; }, []), From d001454836cfed6c403ef136d94c495d242d3bad Mon Sep 17 00:00:00 2001 From: Alex Kahan Date: Thu, 9 Jul 2020 15:04:10 -0400 Subject: [PATCH 2/2] Renaming --- .../server/endpoint/lib/artifacts/lists.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts index e08bd3f135d972..b756c4e3d14c33 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts @@ -100,12 +100,12 @@ export function translateToEndpointExceptions( const entrySet = new Set(); const entriesFiltered: TranslatedExceptionListItem[] = []; if (schemaVersion === 'v1') { - exc.data.forEach((item) => { - const translatedItem = translateItem(schemaVersion, item); - const itemHash = createHash('sha256').update(JSON.stringify(translatedItem)).digest('hex'); - if (!entrySet.has(itemHash)) { + exc.data.forEach((entry) => { + const translatedItem = translateItem(schemaVersion, entry); + const entryHash = createHash('sha256').update(JSON.stringify(translatedItem)).digest('hex'); + if (!entrySet.has(entryHash)) { entriesFiltered.push(translatedItem); - entrySet.add(itemHash); + entrySet.add(entryHash); } }); return entriesFiltered;