From 3fa7834ab8fc36443ea92a8ffdd5f07eff48b4b5 Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Fri, 28 May 2021 13:47:10 -0400 Subject: [PATCH] [Security Solution] Improve find rule and find rule status route performance (#99678) (#100912) * Fetch rule statuses using single aggregation instead of N separate requests * Optimize _find API and _find_statuses * Merge alerting framework errors into rule statuses * Add sortSchema for top hits agg, update terms.order schema Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Marshall Main <55718608+marshallmain@users.noreply.github.com> --- .../aggregations/aggs_types/bucket_aggs.ts | 9 ++- .../aggregations/aggs_types/common_schemas.ts | 29 ++++++++ .../aggregations/aggs_types/metrics_aggs.ts | 3 +- .../routes/__mocks__/request_responses.ts | 66 +++++++++++++++++++ .../routes/rules/find_rules_route.test.ts | 4 +- .../routes/rules/find_rules_route.ts | 46 ++----------- .../rules/find_rules_status_route.test.ts | 4 +- .../routes/rules/find_rules_status_route.ts | 49 +++++--------- .../routes/rules/utils.test.ts | 20 +----- .../detection_engine/routes/rules/utils.ts | 43 +++++------- .../lib/detection_engine/routes/utils.test.ts | 6 +- .../lib/detection_engine/routes/utils.ts | 19 +++--- .../get_bulk_rule_actions_saved_object.ts | 40 +++++++++++ .../lib/detection_engine/rules/types.ts | 6 -- .../schemas/rule_converters.ts | 41 ++++++++---- .../rule_status_saved_objects_client.mock.ts | 1 + .../rule_status_saved_objects_client.ts | 51 ++++++++++++++ .../detection_engine/signals/utils.test.ts | 23 +++++++ .../lib/detection_engine/signals/utils.ts | 15 ++++- 19 files changed, 321 insertions(+), 154 deletions(-) create mode 100644 src/core/server/saved_objects/service/lib/aggregations/aggs_types/common_schemas.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_actions/get_bulk_rule_actions_saved_object.ts diff --git a/src/core/server/saved_objects/service/lib/aggregations/aggs_types/bucket_aggs.ts b/src/core/server/saved_objects/service/lib/aggregations/aggs_types/bucket_aggs.ts index 186962b568792f..cf27505e8f073c 100644 --- a/src/core/server/saved_objects/service/lib/aggregations/aggs_types/bucket_aggs.ts +++ b/src/core/server/saved_objects/service/lib/aggregations/aggs_types/bucket_aggs.ts @@ -7,6 +7,7 @@ */ import { schema as s, ObjectType } from '@kbn/config-schema'; +import { sortOrderSchema } from './common_schemas'; /** * Schemas for the Bucket aggregations. @@ -85,6 +86,12 @@ export const bucketAggsSchemas: Record = { min_doc_count: s.maybe(s.number({ min: 1 })), size: s.maybe(s.number()), show_term_doc_count_error: s.maybe(s.boolean()), - order: s.maybe(s.oneOf([s.literal('asc'), s.literal('desc')])), + order: s.maybe( + s.oneOf([ + sortOrderSchema, + s.recordOf(s.string(), sortOrderSchema), + s.arrayOf(s.recordOf(s.string(), sortOrderSchema)), + ]) + ), }), }; diff --git a/src/core/server/saved_objects/service/lib/aggregations/aggs_types/common_schemas.ts b/src/core/server/saved_objects/service/lib/aggregations/aggs_types/common_schemas.ts new file mode 100644 index 00000000000000..92a3096374687d --- /dev/null +++ b/src/core/server/saved_objects/service/lib/aggregations/aggs_types/common_schemas.ts @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { schema as s } from '@kbn/config-schema'; + +// note: these schemas are not exhaustive. See the `Sort` type of `@elastic/elasticsearch` if you need to enhance it. +const fieldSchema = s.string(); +export const sortOrderSchema = s.oneOf([s.literal('asc'), s.literal('desc'), s.literal('_doc')]); +const sortModeSchema = s.oneOf([ + s.literal('min'), + s.literal('max'), + s.literal('sum'), + s.literal('avg'), + s.literal('median'), +]); +const fieldSortSchema = s.object({ + missing: s.maybe(s.oneOf([s.string(), s.number(), s.boolean()])), + mode: s.maybe(sortModeSchema), + order: s.maybe(sortOrderSchema), + // nested and unmapped_type not implemented yet +}); +const sortContainerSchema = s.recordOf(s.string(), s.oneOf([sortOrderSchema, fieldSortSchema])); +const sortCombinationsSchema = s.oneOf([fieldSchema, sortContainerSchema]); +export const sortSchema = s.oneOf([sortCombinationsSchema, s.arrayOf(sortCombinationsSchema)]); diff --git a/src/core/server/saved_objects/service/lib/aggregations/aggs_types/metrics_aggs.ts b/src/core/server/saved_objects/service/lib/aggregations/aggs_types/metrics_aggs.ts index c05ae67cd2164e..fb7e25fae19f29 100644 --- a/src/core/server/saved_objects/service/lib/aggregations/aggs_types/metrics_aggs.ts +++ b/src/core/server/saved_objects/service/lib/aggregations/aggs_types/metrics_aggs.ts @@ -7,6 +7,7 @@ */ import { schema as s, ObjectType } from '@kbn/config-schema'; +import { sortSchema } from './common_schemas'; /** * Schemas for the metrics Aggregations @@ -68,7 +69,7 @@ export const metricsAggsSchemas: Record = { stored_fields: s.maybe(s.oneOf([s.string(), s.arrayOf(s.string())])), from: s.maybe(s.number()), size: s.maybe(s.number()), - sort: s.maybe(s.oneOf([s.literal('asc'), s.literal('desc')])), + sort: s.maybe(sortSchema), seq_no_primary_term: s.maybe(s.boolean()), version: s.maybe(s.boolean()), track_scores: s.maybe(s.boolean()), diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts index 43377251019176..857762dec45e94 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts @@ -491,6 +491,72 @@ export const getFindResultStatus = (): SavedObjectsFindResponse => ({ + page: 1, + per_page: 6, + total: 2, + saved_objects: [], + aggregations: { + alertIds: { + buckets: [ + { + key: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + most_recent_statuses: { + hits: { + hits: [ + { + _source: { + 'siem-detection-engine-rule-status': { + alertId: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + statusDate: '2020-02-18T15:26:49.783Z', + status: 'succeeded', + lastFailureAt: undefined, + lastSuccessAt: '2020-02-18T15:26:49.783Z', + lastFailureMessage: undefined, + lastSuccessMessage: 'succeeded', + lastLookBackDate: new Date('2020-02-18T15:14:58.806Z').toISOString(), + gap: '500.32', + searchAfterTimeDurations: ['200.00'], + bulkCreateTimeDurations: ['800.43'], + }, + }, + }, + ], + }, + }, + }, + { + key: '1ea5a820-4da1-4e82-92a1-2b43a7bece08', + most_recent_statuses: { + hits: { + hits: [ + { + _source: { + 'siem-detection-engine-rule-status': { + alertId: '1ea5a820-4da1-4e82-92a1-2b43a7bece08', + statusDate: '2020-02-18T15:15:58.806Z', + status: 'failed', + lastFailureAt: '2020-02-18T15:15:58.806Z', + lastSuccessAt: '2020-02-13T20:31:59.855Z', + lastFailureMessage: + 'Signal rule name: "Query with a rule id Number 1", id: "1ea5a820-4da1-4e82-92a1-2b43a7bece08", rule_id: "query-rule-id-1" has a time gap of 5 days (412682928ms), and could be missing signals within that time. Consider increasing your look behind time or adding more Kibana instances.', + lastSuccessMessage: 'succeeded', + lastLookBackDate: new Date('2020-02-18T15:14:58.806Z').toISOString(), + gap: '500.32', + searchAfterTimeDurations: ['200.00'], + bulkCreateTimeDurations: ['800.43'], + }, + }, + }, + ], + }, + }, + }, + ], + }, + }, +}); + export const getEmptySignalsResponse = (): SignalSearchResponse => ({ took: 1, timed_out: false, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rules_route.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rules_route.test.ts index 434ef0f88b1969..06f3ca83c4722a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rules_route.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rules_route.test.ts @@ -10,7 +10,7 @@ import { getAlertMock, getFindRequest, getFindResultWithSingleHit, - getFindResultStatus, + getFindBulkResultStatus, } from '../__mocks__/request_responses'; import { requestContextMock, serverMock, requestMock } from '../__mocks__'; import { findRulesRoute } from './find_rules_route'; @@ -27,7 +27,7 @@ describe('find_rules', () => { clients.alertsClient.find.mockResolvedValue(getFindResultWithSingleHit()); clients.alertsClient.get.mockResolvedValue(getAlertMock(getQueryRuleParams())); - clients.savedObjectsClient.find.mockResolvedValue(getFindResultStatus()); + clients.savedObjectsClient.find.mockResolvedValue(getFindBulkResultStatus()); findRulesRoute(server.router); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rules_route.ts index ccf0a59e87c749..347d005c58a4a8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rules_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rules_route.ts @@ -15,11 +15,10 @@ import type { SecuritySolutionPluginRouter } from '../../../../types'; import { DETECTION_ENGINE_RULES_URL } from '../../../../../common/constants'; import { findRules } from '../../rules/find_rules'; import { buildSiemResponse } from '../utils'; - -import { getRuleActionsSavedObject } from '../../rule_actions/get_rule_actions_saved_object'; import { ruleStatusSavedObjectsClientFactory } from '../../signals/rule_status_saved_objects_client'; import { buildRouteValidation } from '../../../../utils/build_validation/route_validation'; import { transformFindAlerts } from './utils'; +import { getBulkRuleActionsSavedObject } from '../../rule_actions/get_bulk_rule_actions_saved_object'; export const findRulesRoute = (router: SecuritySolutionPluginRouter) => { router.get( @@ -60,44 +59,11 @@ export const findRulesRoute = (router: SecuritySolutionPluginRouter) => { filter: query.filter, fields: query.fields, }); - - // if any rules attempted to execute but failed before the rule executor is called, - // an execution status will be written directly onto the rule via the kibana alerting framework, - // which we are filtering on and will write a failure status - // for any rules found to be in a failing state into our rule status saved objects - const failingRules = rules.data.filter( - (rule) => rule.executionStatus != null && rule.executionStatus.status === 'error' - ); - - const ruleStatuses = await Promise.all( - rules.data.map(async (rule) => { - const results = await ruleStatusClient.find({ - perPage: 1, - sortField: 'statusDate', - sortOrder: 'desc', - search: rule.id, - searchFields: ['alertId'], - }); - const failingRule = failingRules.find((badRule) => badRule.id === rule.id); - if (failingRule != null) { - if (results.saved_objects.length > 0) { - results.saved_objects[0].attributes.status = 'failed'; - results.saved_objects[0].attributes.lastFailureAt = failingRule.executionStatus.lastExecutionDate.toISOString(); - } - } - return results; - }) - ); - const ruleActions = await Promise.all( - rules.data.map(async (rule) => { - const results = await getRuleActionsSavedObject({ - savedObjectsClient, - ruleAlertId: rule.id, - }); - - return results; - }) - ); + const alertIds = rules.data.map((rule) => rule.id); + const [ruleStatuses, ruleActions] = await Promise.all([ + ruleStatusClient.findBulk(alertIds, 1), + getBulkRuleActionsSavedObject({ alertIds, savedObjectsClient }), + ]); const transformed = transformFindAlerts(rules, ruleActions, ruleStatuses); if (transformed == null) { return siemResponse.error({ statusCode: 500, body: 'Internal error transforming' }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rules_status_route.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rules_status_route.test.ts index c3a53a1f393ec5..73f076649b72fe 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rules_status_route.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rules_status_route.test.ts @@ -7,9 +7,9 @@ import { DETECTION_ENGINE_RULES_URL } from '../../../../../common/constants'; import { - getFindResultStatus, ruleStatusRequest, getAlertMock, + getFindBulkResultStatus, } from '../__mocks__/request_responses'; import { serverMock, requestContextMock, requestMock } from '../__mocks__'; import { findRulesStatusesRoute } from './find_rules_status_route'; @@ -26,7 +26,7 @@ describe('find_statuses', () => { beforeEach(async () => { server = serverMock.create(); ({ clients, context } = requestContextMock.createTools()); - clients.savedObjectsClient.find.mockResolvedValue(getFindResultStatus()); // successful status search + clients.savedObjectsClient.find.mockResolvedValue(getFindBulkResultStatus()); // successful status search clients.alertsClient.get.mockResolvedValue(getAlertMock(getQueryRuleParams())); findRulesStatusesRoute(server.router); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rules_status_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rules_status_route.ts index bd6e8fc9e7aadb..aed8b80e4f1332 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rules_status_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rules_status_route.ts @@ -9,14 +9,13 @@ import { transformError } from '@kbn/securitysolution-es-utils'; import { buildRouteValidation } from '../../../../utils/build_validation/route_validation'; import type { SecuritySolutionPluginRouter } from '../../../../types'; import { DETECTION_ENGINE_RULES_URL } from '../../../../../common/constants'; -import { RuleStatusResponse } from '../../rules/types'; import { buildSiemResponse, mergeStatuses, getFailingRules } from '../utils'; - import { ruleStatusSavedObjectsClientFactory } from '../../signals/rule_status_saved_objects_client'; import { findRulesStatusesSchema, FindRulesStatusesSchemaDecoded, } from '../../../../../common/detection_engine/schemas/request/find_rule_statuses_schema'; +import { mergeAlertWithSidecarStatus } from '../../schemas/rule_converters'; /** * Given a list of rule ids, return the current status and @@ -51,45 +50,27 @@ export const findRulesStatusesRoute = (router: SecuritySolutionPluginRouter) => const ids = body.ids; try { const ruleStatusClient = ruleStatusSavedObjectsClientFactory(savedObjectsClient); - const failingRules = await getFailingRules(ids, alertsClient); + const [statusesById, failingRules] = await Promise.all([ + ruleStatusClient.findBulk(ids, 6), + getFailingRules(ids, alertsClient), + ]); - const statuses = await ids.reduce(async (acc, id) => { - const accumulated = await acc; - const lastFiveErrorsForId = await ruleStatusClient.find({ - perPage: 6, - sortField: 'statusDate', - sortOrder: 'desc', - search: id, - searchFields: ['alertId'], - }); + const statuses = ids.reduce((acc, id) => { + const lastFiveErrorsForId = statusesById[id]; - if (lastFiveErrorsForId.saved_objects.length === 0) { - return accumulated; + if (lastFiveErrorsForId == null || lastFiveErrorsForId.length === 0) { + return acc; } const failingRule = failingRules[id]; - const lastFailureAt = lastFiveErrorsForId.saved_objects[0].attributes.lastFailureAt; - - if ( - failingRule != null && - (lastFailureAt == null || - new Date(failingRule.executionStatus.lastExecutionDate) > new Date(lastFailureAt)) - ) { - const currentStatus = lastFiveErrorsForId.saved_objects[0]; - currentStatus.attributes.lastFailureMessage = `Reason: ${failingRule.executionStatus.error?.reason} Message: ${failingRule.executionStatus.error?.message}`; - currentStatus.attributes.lastFailureAt = failingRule.executionStatus.lastExecutionDate.toISOString(); - currentStatus.attributes.statusDate = failingRule.executionStatus.lastExecutionDate.toISOString(); - currentStatus.attributes.status = 'failed'; - const updatedLastFiveErrorsSO = [ - currentStatus, - ...lastFiveErrorsForId.saved_objects.slice(1), - ]; - return mergeStatuses(id, updatedLastFiveErrorsSO, accumulated); + if (failingRule != null) { + const currentStatus = mergeAlertWithSidecarStatus(failingRule, lastFiveErrorsForId[0]); + const updatedLastFiveErrorsSO = [currentStatus, ...lastFiveErrorsForId.slice(1)]; + return mergeStatuses(id, updatedLastFiveErrorsSO, acc); } - return mergeStatuses(id, [...lastFiveErrorsForId.saved_objects], accumulated); - }, Promise.resolve({})); - + return mergeStatuses(id, [...lastFiveErrorsForId], acc); + }, {}); return response.ok({ body: statuses }); } catch (err) { const error = transformError(err); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.test.ts index f2788ab1bd4c99..29e322d7fcab5f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.test.ts @@ -27,7 +27,6 @@ import { PartialFilter } from '../../types'; import { BulkError, ImportSuccessError } from '../utils'; import { getOutputRuleAlertForRest } from '../__mocks__/utils'; import { PartialAlert } from '../../../../../../alerting/server'; -import { SanitizedAlert } from '../../../../../../alerting/server/types'; import { createRulesStreamFromNdJson } from '../../rules/create_rules_stream_from_ndjson'; import { RuleAlertType } from '../../rules/types'; import { ImportRulesSchemaDecoded } from '../../../../../common/detection_engine/schemas/request/import_rules_schema'; @@ -256,7 +255,7 @@ describe('utils', () => { describe('transformFindAlerts', () => { test('outputs empty data set when data set is empty correct', () => { - const output = transformFindAlerts({ data: [], page: 1, perPage: 0, total: 0 }, []); + const output = transformFindAlerts({ data: [], page: 1, perPage: 0, total: 0 }, {}, {}); expect(output).toEqual({ data: [], page: 1, perPage: 0, total: 0 }); }); @@ -268,7 +267,8 @@ describe('utils', () => { total: 0, data: [getAlertMock(getQueryRuleParams())], }, - [] + {}, + {} ); const expected = getOutputRuleAlertForRest(); expect(output).toEqual({ @@ -278,20 +278,6 @@ describe('utils', () => { data: [expected], }); }); - - test('returns 500 if the data is not of type siem alert', () => { - const unsafeCast = ([{ name: 'something else' }] as unknown) as SanitizedAlert[]; - const output = transformFindAlerts( - { - data: unsafeCast, - page: 1, - perPage: 1, - total: 1, - }, - [] - ); - expect(output).toBeNull(); - }); }); describe('transform', () => { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.ts index 466b8dd1842276..dc0cd2e497215f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/utils.ts @@ -6,7 +6,7 @@ */ import { countBy } from 'lodash/fp'; -import { SavedObject, SavedObjectsFindResponse } from 'kibana/server'; +import { SavedObject } from 'kibana/server'; import uuid from 'uuid'; import { RulesSchema } from '../../../../../common/detection_engine/schemas/response/rules_schema'; @@ -17,11 +17,10 @@ import { INTERNAL_IDENTIFIER } from '../../../../../common/constants'; import { RuleAlertType, isAlertType, - isAlertTypes, IRuleSavedAttributesSavedObjectAttributes, isRuleStatusFindType, - isRuleStatusFindTypes, isRuleStatusSavedObjectType, + IRuleStatusSOAttributes, } from '../../rules/types'; import { createBulkErrorObject, @@ -34,6 +33,7 @@ import { import { RuleActions } from '../../rule_actions/types'; import { internalRuleToAPIResponse } from '../../schemas/rule_converters'; import { RuleParams } from '../../schemas/rule_schemas'; +import { SanitizedAlert } from '../../../../../../alerting/common'; type PromiseFromStreams = ImportRulesSchemaDecoded | Error; @@ -103,11 +103,11 @@ export const transformTags = (tags: string[]): string[] => { // Transforms the data but will remove any null or undefined it encounters and not include // those on the export export const transformAlertToRule = ( - alert: RuleAlertType, + alert: SanitizedAlert, ruleActions?: RuleActions | null, ruleStatus?: SavedObject ): Partial => { - return internalRuleToAPIResponse(alert, ruleActions, ruleStatus); + return internalRuleToAPIResponse(alert, ruleActions, ruleStatus?.attributes); }; export const transformAlertsToRules = (alerts: RuleAlertType[]): Array> => { @@ -116,33 +116,24 @@ export const transformAlertsToRules = (alerts: RuleAlertType[]): Array, - ruleActions: Array, - ruleStatuses?: Array> + ruleActions: { [key: string]: RuleActions | undefined }, + ruleStatuses: { [key: string]: IRuleStatusSOAttributes[] | undefined } ): { page: number; perPage: number; total: number; data: Array>; } | null => { - if (!ruleStatuses && isAlertTypes(findResults.data)) { - return { - page: findResults.page, - perPage: findResults.perPage, - total: findResults.total, - data: findResults.data.map((alert, idx) => transformAlertToRule(alert, ruleActions[idx])), - }; - } else if (isAlertTypes(findResults.data) && isRuleStatusFindTypes(ruleStatuses)) { - return { - page: findResults.page, - perPage: findResults.perPage, - total: findResults.total, - data: findResults.data.map((alert, idx) => - transformAlertToRule(alert, ruleActions[idx], ruleStatuses[idx].saved_objects[0]) - ), - }; - } else { - return null; - } + return { + page: findResults.page, + perPage: findResults.perPage, + total: findResults.total, + data: findResults.data.map((alert) => { + const statuses = ruleStatuses[alert.id]; + const status = statuses ? statuses[0] : undefined; + return internalRuleToAPIResponse(alert, ruleActions[alert.id], status); + }), + }; }; export const transform = ( diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/utils.test.ts index a09c4cd257618d..ce7d4b31733700 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/utils.test.ts @@ -25,7 +25,7 @@ import { getFailingRules, } from './utils'; import { responseMock } from './__mocks__'; -import { exampleRuleStatus, exampleFindRuleStatusResponse } from '../signals/__mocks__/es_results'; +import { exampleRuleStatus } from '../signals/__mocks__/es_results'; import { getAlertMock } from './__mocks__/request_responses'; import { AlertExecutionStatusErrorReasons } from '../../../../../alerting/common'; import { getQueryRuleParams } from '../schemas/rule_schemas.mock'; @@ -301,8 +301,8 @@ describe('utils', () => { const statusTwo = exampleRuleStatus(); statusTwo.attributes.status = 'failed'; const currentStatus = exampleRuleStatus(); - const foundRules = exampleFindRuleStatusResponse([currentStatus, statusOne, statusTwo]); - const res = mergeStatuses(currentStatus.attributes.alertId, foundRules.saved_objects, { + const foundRules = [currentStatus.attributes, statusOne.attributes, statusTwo.attributes]; + const res = mergeStatuses(currentStatus.attributes.alertId, foundRules, { 'myfakealertid-8cfac': { current_status: { alert_id: 'myfakealertid-8cfac', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/utils.ts index 130084da21591b..9ff75726322a17 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/utils.ts @@ -14,11 +14,12 @@ import { RouteValidationFunction, KibanaResponseFactory, CustomHttpResponseOptions, - SavedObjectsFindResult, } from '../../../../../../../src/core/server'; import { AlertsClient } from '../../../../../alerting/server'; import { RuleStatusResponse, IRuleStatusSOAttributes } from '../rules/types'; +import { RuleParams } from '../schemas/rule_schemas'; + export interface OutputError { message: string; statusCode: number; @@ -277,7 +278,7 @@ export const convertToSnakeCase = >( */ export const mergeStatuses = ( id: string, - currentStatusAndFailures: Array>, + currentStatusAndFailures: IRuleStatusSOAttributes[], acc: RuleStatusResponse ): RuleStatusResponse => { if (currentStatusAndFailures.length === 0) { @@ -286,7 +287,7 @@ export const mergeStatuses = ( }; } const convertedCurrentStatus = convertToSnakeCase( - currentStatusAndFailures[0].attributes + currentStatusAndFailures[0] ); return { ...acc, @@ -294,12 +295,12 @@ export const mergeStatuses = ( current_status: convertedCurrentStatus, failures: currentStatusAndFailures .slice(1) - .map((errorItem) => convertToSnakeCase(errorItem.attributes)), + .map((errorItem) => convertToSnakeCase(errorItem)), }, } as RuleStatusResponse; }; -export type GetFailingRulesResult = Record; +export type GetFailingRulesResult = Record>; export const getFailingRules = async ( ids: string[], @@ -316,13 +317,11 @@ export const getFailingRules = async ( return errorRules .filter((rule) => rule.executionStatus.status === 'error') .reduce((acc, failingRule) => { - const accum = acc; - const theRule = failingRule; return { - [theRule.id]: { - ...theRule, + [failingRule.id]: { + ...failingRule, }, - ...accum, + ...acc, }; }, {}); } catch (exc) { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_actions/get_bulk_rule_actions_saved_object.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_actions/get_bulk_rule_actions_saved_object.ts new file mode 100644 index 00000000000000..1abb16ba4612c1 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_actions/get_bulk_rule_actions_saved_object.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { AlertServices } from '../../../../../alerting/server'; +import { ruleActionsSavedObjectType } from './saved_object_mappings'; +import { IRuleActionsAttributesSavedObjectAttributes } from './types'; +import { getRuleActionsFromSavedObject } from './utils'; +import { RulesActionsSavedObject } from './get_rule_actions_saved_object'; +import { buildChunkedOrFilter } from '../signals/utils'; + +interface GetBulkRuleActionsSavedObject { + alertIds: string[]; + savedObjectsClient: AlertServices['savedObjectsClient']; +} + +export const getBulkRuleActionsSavedObject = async ({ + alertIds, + savedObjectsClient, +}: GetBulkRuleActionsSavedObject): Promise> => { + const filter = buildChunkedOrFilter( + `${ruleActionsSavedObjectType}.attributes.ruleAlertId`, + alertIds + ); + const { + // eslint-disable-next-line @typescript-eslint/naming-convention + saved_objects, + } = await savedObjectsClient.find({ + type: ruleActionsSavedObjectType, + perPage: 10000, + filter, + }); + return saved_objects.reduce((acc: { [key: string]: RulesActionsSavedObject }, savedObject) => { + acc[savedObject.attributes.ruleAlertId] = getRuleActionsFromSavedObject(savedObject); + return acc; + }, {}); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts index 380eb085e0d5a8..601f3ebaa0f9e6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts @@ -211,12 +211,6 @@ export const isRuleStatusFindType = ( return get('saved_objects', obj) != null; }; -export const isRuleStatusFindTypes = ( - obj: unknown[] | undefined -): obj is Array> => { - return obj ? obj.every((ruleStatus) => isRuleStatusFindType(ruleStatus)) : false; -}; - export interface CreateRulesOptions { alertsClient: AlertsClient; anomalyThreshold: AnomalyThresholdOrUndefined; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/schemas/rule_converters.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/schemas/rule_converters.ts index ee7ecaadfd95c0..a215da021d15ad 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/schemas/rule_converters.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/schemas/rule_converters.ts @@ -6,7 +6,6 @@ */ import uuid from 'uuid'; -import { SavedObject } from 'kibana/server'; import { normalizeMachineLearningJobIds, normalizeThresholdObject, @@ -29,8 +28,8 @@ import { AppClient } from '../../../types'; import { addTags } from '../rules/add_tags'; import { DEFAULT_MAX_SIGNALS, SERVER_APP_ID, SIGNALS_ID } from '../../../../common/constants'; import { transformRuleToAlertAction } from '../../../../common/detection_engine/transform_actions'; -import { Alert } from '../../../../../alerting/common'; -import { IRuleSavedAttributesSavedObjectAttributes } from '../rules/types'; +import { SanitizedAlert } from '../../../../../alerting/common'; +import { IRuleStatusSOAttributes } from '../rules/types'; import { transformTags } from '../routes/rules/utils'; // These functions provide conversions from the request API schema to the internal rule schema and from the internal rule schema @@ -270,10 +269,11 @@ export const commonParamsCamelToSnake = (params: BaseRuleParams) => { }; export const internalRuleToAPIResponse = ( - rule: Alert, + rule: SanitizedAlert, ruleActions?: RuleActions | null, - ruleStatus?: SavedObject + ruleStatus?: IRuleStatusSOAttributes ): FullResponseSchema => { + const mergedStatus = ruleStatus ? mergeAlertWithSidecarStatus(rule, ruleStatus) : undefined; return { // Alerting framework params id: rule.id, @@ -293,11 +293,30 @@ export const internalRuleToAPIResponse = ( throttle: ruleActions?.ruleThrottle || 'no_actions', actions: ruleActions?.actions ?? [], // Rule status - status: ruleStatus?.attributes.status ?? undefined, - status_date: ruleStatus?.attributes.statusDate ?? undefined, - last_failure_at: ruleStatus?.attributes.lastFailureAt ?? undefined, - last_success_at: ruleStatus?.attributes.lastSuccessAt ?? undefined, - last_failure_message: ruleStatus?.attributes.lastFailureMessage ?? undefined, - last_success_message: ruleStatus?.attributes.lastSuccessMessage ?? undefined, + status: mergedStatus?.status ?? undefined, + status_date: mergedStatus?.statusDate ?? undefined, + last_failure_at: mergedStatus?.lastFailureAt ?? undefined, + last_success_at: mergedStatus?.lastSuccessAt ?? undefined, + last_failure_message: mergedStatus?.lastFailureMessage ?? undefined, + last_success_message: mergedStatus?.lastSuccessMessage ?? undefined, }; }; + +export const mergeAlertWithSidecarStatus = ( + alert: SanitizedAlert, + status: IRuleStatusSOAttributes +): IRuleStatusSOAttributes => { + if ( + new Date(alert.executionStatus.lastExecutionDate) > new Date(status.statusDate) && + alert.executionStatus.status === 'error' + ) { + return { + ...status, + lastFailureMessage: `Reason: ${alert.executionStatus.error?.reason} Message: ${alert.executionStatus.error?.message}`, + lastFailureAt: alert.executionStatus.lastExecutionDate.toISOString(), + statusDate: alert.executionStatus.lastExecutionDate.toISOString(), + status: 'failed', + }; + } + return status; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/rule_status_saved_objects_client.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/rule_status_saved_objects_client.mock.ts index b760cec9226db0..3dd328a9499386 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/rule_status_saved_objects_client.mock.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/rule_status_saved_objects_client.mock.ts @@ -9,6 +9,7 @@ import { RuleStatusSavedObjectsClient } from '../rule_status_saved_objects_clien const createMockRuleStatusSavedObjectsClient = (): jest.Mocked => ({ find: jest.fn(), + findBulk: jest.fn(), create: jest.fn(), update: jest.fn(), delete: jest.fn(), diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/rule_status_saved_objects_client.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/rule_status_saved_objects_client.ts index ff1b0e27019edb..25d315279ad60f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/rule_status_saved_objects_client.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/rule_status_saved_objects_client.ts @@ -5,6 +5,7 @@ * 2.0. */ +import { get } from 'lodash'; import { SavedObjectsClientContract, SavedObject, @@ -14,11 +15,13 @@ import { } from '../../../../../../../src/core/server'; import { ruleStatusSavedObjectType } from '../rules/saved_object_mappings'; import { IRuleStatusSOAttributes } from '../rules/types'; +import { buildChunkedOrFilter } from './utils'; export interface RuleStatusSavedObjectsClient { find: ( options?: Omit ) => Promise>; + findBulk: (ids: string[], statusesPerId: number) => Promise; create: (attributes: IRuleStatusSOAttributes) => Promise>; update: ( id: string, @@ -27,6 +30,10 @@ export interface RuleStatusSavedObjectsClient { delete: (id: string) => Promise<{}>; } +interface FindBulkResponse { + [key: string]: IRuleStatusSOAttributes[] | undefined; +} + export const ruleStatusSavedObjectsClientFactory = ( savedObjectsClient: SavedObjectsClientContract ): RuleStatusSavedObjectsClient => ({ @@ -35,6 +42,50 @@ export const ruleStatusSavedObjectsClientFactory = ( ...options, type: ruleStatusSavedObjectType, }), + findBulk: async (ids, statusesPerId) => { + if (ids.length === 0) { + return {}; + } + const filter = buildChunkedOrFilter(`${ruleStatusSavedObjectType}.attributes.alertId`, ids); + const order: 'desc' = 'desc'; + const aggs = { + alertIds: { + terms: { + field: `${ruleStatusSavedObjectType}.attributes.alertId`, + size: ids.length, + }, + aggs: { + most_recent_statuses: { + top_hits: { + sort: [ + { + [`${ruleStatusSavedObjectType}.statusDate`]: { + order, + }, + }, + ], + size: statusesPerId, + }, + }, + }, + }, + }; + const results = await savedObjectsClient.find({ + filter, + aggs, + type: ruleStatusSavedObjectType, + perPage: 0, + }); + const buckets = get(results, 'aggregations.alertIds.buckets'); + return buckets.reduce((acc: Record, bucket: unknown) => { + const key = get(bucket, 'key'); + const hits = get(bucket, 'most_recent_statuses.hits.hits'); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const statuses = hits.map((hit: any) => hit._source['siem-detection-engine-rule-status']); + acc[key] = statuses; + return acc; + }, {}); + }, create: (attributes) => savedObjectsClient.create(ruleStatusSavedObjectType, attributes), update: (id, attributes) => savedObjectsClient.update(ruleStatusSavedObjectType, id, attributes), delete: (id) => savedObjectsClient.delete(ruleStatusSavedObjectType, id), diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts index b04eab1496e960..f49492939eeb10 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts @@ -39,6 +39,7 @@ import { createTotalHitsFromSearchResult, lastValidDate, calculateThresholdSignalUuid, + buildChunkedOrFilter, } from './utils'; import { BulkResponseErrorAggregation, SearchAfterAndBulkCreateReturnType } from './types'; import { @@ -1473,4 +1474,26 @@ describe('utils', () => { expect(signalUuid).toEqual('ee8870dc-45ff-5e6c-a2f9-80886651ce03'); }); }); + + describe('buildChunkedOrFilter', () => { + test('should return undefined if no values are provided', () => { + const filter = buildChunkedOrFilter('field.name', []); + expect(filter).toEqual(undefined); + }); + + test('should return a filter with a single value', () => { + const filter = buildChunkedOrFilter('field.name', ['id-1']); + expect(filter).toEqual('field.name: ("id-1")'); + }); + + test('should return a filter with a multiple values', () => { + const filter = buildChunkedOrFilter('field.name', ['id-1', 'id-2']); + expect(filter).toEqual('field.name: ("id-1" OR "id-2")'); + }); + + test('should return a filter with a multiple values chunked', () => { + const filter = buildChunkedOrFilter('field.name', ['id-1', 'id-2', 'id-3'], 2); + expect(filter).toEqual('field.name: ("id-1" OR "id-2") OR field.name: ("id-3")'); + }); + }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts index 03a067af6066d4..cc4ed6a45807b1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts @@ -10,7 +10,7 @@ import moment from 'moment'; import uuidv5 from 'uuid/v5'; import dateMath from '@elastic/datemath'; import type { estypes } from '@elastic/elasticsearch'; -import { isEmpty, partition } from 'lodash'; +import { chunk, isEmpty, partition } from 'lodash'; import { ApiResponse, Context } from '@elastic/elasticsearch/lib/Transport'; import { SortResults } from '@elastic/elasticsearch/api/types'; @@ -868,3 +868,16 @@ export const getSafeSortIds = (sortIds: SortResults | undefined) => { return sortId; }); }; + +export const buildChunkedOrFilter = (field: string, values: string[], chunkSize: number = 1024) => { + if (values.length === 0) { + return undefined; + } + const chunkedValues = chunk(values, chunkSize); + return chunkedValues + .map((subArray) => { + const joinedValues = subArray.map((value) => `"${value}"`).join(' OR '); + return `${field}: (${joinedValues})`; + }) + .join(' OR '); +};