[RAC][Security Solution] Adds migration to new SecuritySolution rule …

…types (#112113) * Initial commit * Properly handle signal history * Fix #95258 - cardinality sort bug * Init threshold rule * Create working threshold rule * Fix threshold signal generation * Fix tests * Update mappings * ALERT_TYPE_ID => RULE_TYPE_ID * Add tests * Fix types * Adds RAC rule type migration * Fix threshold tests (remove outputIndex) * Add threshold rule type to ruleTypeMappings * Add kbn-securitysolution-rules package for sharing with alerting framework * Fix type errors * Fix find_rules tests * First round of test fixes * Fix issues from merge conflicts * Use ruleDataClient getReader() for reading * Fixes to 'generating_signals' tests * Remove more refs to legacy schema * Linting * Quick type fix * Bug fixes * Add saved query rule type * Linting * Fix types * Signal generation tests * Test updates * Update some more refs * build_alert tests * Cleanup * Ref updates * Revert "Ref updates" This reverts commit 4d1473d. * Update status field * Test fixes * Another test * Got a little too aggressive with search/replace * let's see where we're at * Fix * Test fixes * cleanup * Fix cases API integration test config, flaky DE tests * Move flattenWithPrefix to package / skip signal migration tests * Fix unit tests * Use new schema for bulk rule creation * event: { kind } => event.kind * Fix signal migration API tests * Fix ml integration test * Fix threat match integration tests * Fix ML rule type tests and add correct producer to all rule types * Update threat match API integration test * Remove dupe properties * Type fix * Fix ML producer in functional test * Fix generating_signals tests * Remove usage of RuleDataClient-based execution log client * Don't check output index version if rule registry enabled * Fix bulk duplicate rule * Fix duplicate rule test * Fix readPrivileges and timestamp check logic * Fixes for eql and exceptions tests... disable open_close_signals * Type fixes / keyword test fixes * Additional test fixes * Unit test fixes + signal -> kibana.alert * Test fixes for exceptions * Fix read_resolve_rules test * Various test fixes with marshallmain * Sort search results * Fix create_rules tests * Disable writer cache for integration tests * Disable writer cache for cases integration tests * Fix types in rule_data_plugin_service * Fix ordering in exceptions tests * Remove rule_registry.enabled flag * Fix signals migration tests * Don't check signals index before creation * Fix cypress config * Fix type error * create_migrations tests * Skip flaky test * Helpful comment * Fixes from merge conflicts * Pretend that signals index exists * Fix type errors * Skip flaky tests * Fix threat matching test * Clean up * Reverting default ruleRegistry experimental flag (breaks unit tests) * Reenable rule registry experimental feature by default * Execute DE rule migration in 8.0 Co-authored-by: Marshall Main <[email protected]>
elastic · Oct 26, 2021 · 117efdf · 117efdf
1 parent f92cbbc
commit 117efdf
Show file tree

Hide file tree

Showing 188 changed files with 2,620 additions and 4,596 deletions.
diff --git a/docs/developer/getting-started/monorepo-packages.asciidoc b/docs/developer/getting-started/monorepo-packages.asciidoc
@@ -91,6 +91,7 @@ yarn kbn watch
 - @kbn/securitysolution-list-constants
 - @kbn/securitysolution-list-hooks
 - @kbn/securitysolution-list-utils
+- @kbn/securitysolution-rules
 - @kbn/securitysolution-utils
 - @kbn/server-http-tools
 - @kbn/server-route-repository

diff --git a/package.json b/package.json
@@ -148,6 +148,7 @@
  "@kbn/securitysolution-list-constants": "link:bazel-bin/packages/kbn-securitysolution-list-constants",
  "@kbn/securitysolution-list-hooks": "link:bazel-bin/packages/kbn-securitysolution-list-hooks",
  "@kbn/securitysolution-list-utils": "link:bazel-bin/packages/kbn-securitysolution-list-utils",
+ "@kbn/securitysolution-rules": "link:bazel-bin/packages/kbn-securitysolution-rules",
  "@kbn/securitysolution-t-grid": "link:bazel-bin/packages/kbn-securitysolution-t-grid",
  "@kbn/securitysolution-utils": "link:bazel-bin/packages/kbn-securitysolution-utils",
  "@kbn/server-http-tools": "link:bazel-bin/packages/kbn-server-http-tools",

diff --git a/packages/BUILD.bazel b/packages/BUILD.bazel
@@ -46,6 +46,7 @@ filegroup(
  "//packages/kbn-securitysolution-list-api:build",
  "//packages/kbn-securitysolution-list-hooks:build",
  "//packages/kbn-securitysolution-list-utils:build",
+ "//packages/kbn-securitysolution-rules:build",
  "//packages/kbn-securitysolution-utils:build",
  "//packages/kbn-securitysolution-es-utils:build",
  "//packages/kbn-securitysolution-t-grid:build",

diff --git a/packages/kbn-rule-data-utils/src/technical_field_names.ts b/packages/kbn-rule-data-utils/src/technical_field_names.ts
@@ -17,6 +17,7 @@ const CONSUMERS = `${KIBANA_NAMESPACE}.consumers` as const;
 const ECS_VERSION = 'ecs.version' as const;
 const EVENT_ACTION = 'event.action' as const;
 const EVENT_KIND = 'event.kind' as const;
+const EVENT_MODULE = 'event.module' as const;
 const SPACE_IDS = `${KIBANA_NAMESPACE}.space_ids` as const;
 const TAGS = 'tags' as const;
 const TIMESTAMP = '@timestamp' as const;
@@ -88,6 +89,7 @@ const fields = {
  ECS_VERSION,
  EVENT_KIND,
  EVENT_ACTION,
+ EVENT_MODULE,
  TAGS,
  TIMESTAMP,
  ALERT_ACTION_GROUP,
@@ -189,6 +191,7 @@ export {
  ECS_VERSION,
  EVENT_ACTION,
  EVENT_KIND,
+ EVENT_MODULE,
  KIBANA_NAMESPACE,
  ALERT_RULE_UUID,
  ALERT_RULE_CATEGORY,

diff --git a/packages/kbn-securitysolution-rules/BUILD.bazel b/packages/kbn-securitysolution-rules/BUILD.bazel
@@ -0,0 +1,95 @@
+load("@npm//@bazel/typescript:index.bzl", "ts_config", "ts_project")
+load("@build_bazel_rules_nodejs//:index.bzl", "js_library", "pkg_npm")
+load("//src/dev/bazel:index.bzl", "jsts_transpiler")
+
+PKG_BASE_NAME = "kbn-securitysolution-rules"
+
+PKG_REQUIRE_NAME = "@kbn/securitysolution-rules"
+
+SOURCE_FILES = glob(
+ [
+ "src/**/*.ts",
+ ],
+ exclude = [
+ "**/*.test.*",
+ "**/*.mock.*",
+ ],
+)
+
+SRCS = SOURCE_FILES
+
+filegroup(
+ name = "srcs",
+ srcs = SRCS,
+)
+
+NPM_MODULE_EXTRA_FILES = [
+ "package.json",
+ "README.md",
+]
+
+RUNTIME_DEPS = [
+ "@npm//lodash",
+ "@npm//tslib",
+ "@npm//uuid",
+]
+
+TYPES_DEPS = [
+ "@npm//tslib",
+ "@npm//@types/jest",
+ "@npm//@types/lodash",
+ "@npm//@types/node",
+ "@npm//@types/uuid"
+]
+
+jsts_transpiler(
+ name = "target_node",
+ srcs = SRCS,
+ build_pkg_name = package_name(),
+)
+
+ts_config(
+ name = "tsconfig",
+ src = "tsconfig.json",
+ deps = [
+ "//:tsconfig.base.json",
+ "//:tsconfig.bazel.json",
+ ],
+)
+
+ts_project(
+ name = "tsc_types",
+ args = ["--pretty"],
+ srcs = SRCS,
+ deps = TYPES_DEPS,
+ declaration = True,
+ declaration_map = True,
+ emit_declaration_only = True,
+ out_dir = "target_types",
+ root_dir = "src",
+ source_map = True,
+ tsconfig = ":tsconfig",
+)
+
+js_library(
+ name = PKG_BASE_NAME,
+ srcs = NPM_MODULE_EXTRA_FILES,
+ deps = RUNTIME_DEPS + [":target_node", ":tsc_types"],
+ package_name = PKG_REQUIRE_NAME,
+ visibility = ["//visibility:public"],
+)
+
+pkg_npm(
+ name = "npm_module",
+ deps = [
+ ":%s" % PKG_BASE_NAME,
+ ],
+)
+
+filegroup(
+ name = "build",
+ srcs = [
+ ":npm_module",
+ ],
+ visibility = ["//visibility:public"],
+)
diff --git a/packages/kbn-securitysolution-rules/README.md b/packages/kbn-securitysolution-rules/README.md
@@ -0,0 +1,3 @@
+# kbn-securitysolution-rules
+
+This contains alerts-as-data rule-specific constants and mappings that can be used across plugins.
diff --git a/packages/kbn-securitysolution-rules/jest.config.js b/packages/kbn-securitysolution-rules/jest.config.js
@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+module.exports = {
+ preset: '@kbn/test',
+ rootDir: '../..',
+ roots: ['<rootDir>/packages/kbn-securitysolution-rules'],
+};
diff --git a/packages/kbn-securitysolution-rules/package.json b/packages/kbn-securitysolution-rules/package.json
@@ -0,0 +1,9 @@
+{
+ "name": "@kbn/securitysolution-rules",
+ "version": "1.0.0",
+ "description": "security solution rule utilities to use across plugins",
+ "license": "SSPL-1.0 OR Elastic License 2.0",
+ "main": "./target_node/index.js",
+ "types": "./target_types/index.d.ts",
+ "private": true
+}
diff --git a/packages/kbn-securitysolution-rules/src/index.ts b/packages/kbn-securitysolution-rules/src/index.ts
@@ -0,0 +1,11 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+export * from './rule_type_constants';
+export * from './rule_type_mappings';
+export * from './utils';
diff --git a/packages/kbn-securitysolution-rules/src/rule_type_constants.ts b/packages/kbn-securitysolution-rules/src/rule_type_constants.ts
@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+/**
+ * Id for the legacy siem signals alerting type
+ */
+export const SIGNALS_ID = `siem.signals` as const;
+
+/**
+ * IDs for alerts-as-data rule types
+ */
+const RULE_TYPE_PREFIX = `siem` as const;
+export const EQL_RULE_TYPE_ID = `${RULE_TYPE_PREFIX}.eqlRule` as const;
+export const INDICATOR_RULE_TYPE_ID = `${RULE_TYPE_PREFIX}.indicatorRule` as const;
+export const ML_RULE_TYPE_ID = `${RULE_TYPE_PREFIX}.mlRule` as const;
+export const QUERY_RULE_TYPE_ID = `${RULE_TYPE_PREFIX}.queryRule` as const;
+export const SAVED_QUERY_RULE_TYPE_ID = `${RULE_TYPE_PREFIX}.savedQueryRule` as const;
+export const THRESHOLD_RULE_TYPE_ID = `${RULE_TYPE_PREFIX}.thresholdRule` as const;
diff --git a/packages/kbn-securitysolution-rules/src/rule_type_mappings.ts b/packages/kbn-securitysolution-rules/src/rule_type_mappings.ts
@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import {
+ EQL_RULE_TYPE_ID,
+ INDICATOR_RULE_TYPE_ID,
+ ML_RULE_TYPE_ID,
+ QUERY_RULE_TYPE_ID,
+ SAVED_QUERY_RULE_TYPE_ID,
+ THRESHOLD_RULE_TYPE_ID,
+} from './rule_type_constants';
+
+/**
+ * Maps legacy rule types to RAC rule type IDs.
+ */
+export const ruleTypeMappings = {
+ eql: EQL_RULE_TYPE_ID,
+ machine_learning: ML_RULE_TYPE_ID,
+ query: QUERY_RULE_TYPE_ID,
+ saved_query: SAVED_QUERY_RULE_TYPE_ID,
+ threat_match: INDICATOR_RULE_TYPE_ID,
+ threshold: THRESHOLD_RULE_TYPE_ID,
+};
+type RuleTypeMappings = typeof ruleTypeMappings;
+
+export type RuleType = keyof RuleTypeMappings;
+export type RuleTypeId = RuleTypeMappings[keyof RuleTypeMappings];
diff --git a/...es/factories/utils/flatten_with_prefix.ts → ...s/kbn-securitysolution-rules/src/utils.ts b/...es/factories/utils/flatten_with_prefix.ts → ...s/kbn-securitysolution-rules/src/utils.ts
@@ -1,12 +1,23 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
  * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
  */
 
 import { isPlainObject } from 'lodash';
-import { SearchTypes } from '../../../../../../common/detection_engine/types';
+import { RuleType, RuleTypeId, ruleTypeMappings } from './rule_type_mappings';
+
+export const isRuleType = (ruleType: unknown): ruleType is RuleType => {
+ return Object.keys(ruleTypeMappings).includes(ruleType as string);
+};
+
+export const isRuleTypeId = (ruleTypeId: unknown): ruleTypeId is RuleTypeId => {
+ return Object.values(ruleTypeMappings).includes(ruleTypeId as RuleTypeId);
+};
+
+type SearchTypes = string | number | boolean | object | SearchTypes[] | undefined;
 
 export const flattenWithPrefix = (
  prefix: string,

diff --git a/packages/kbn-securitysolution-rules/tsconfig.json b/packages/kbn-securitysolution-rules/tsconfig.json
@@ -0,0 +1,19 @@
+{
+ "extends": "../../tsconfig.bazel.json",
+ "compilerOptions": {
+ "declaration": true,
+ "declarationMap": true,
+ "emitDeclarationOnly": true,
+ "outDir": "target_types",
+ "rootDir": "src",
+ "sourceMap": true,
+ "sourceRoot": "../../../../packages/kbn-securitysolution-rules/src",
+ "types": [
+ "jest",
+ "node"
+ ]
+ },
+ "include": [
+ "src/**/*"
+ ]
+}
diff --git a/x-pack/plugins/alerting/server/saved_objects/migrations.ts b/x-pack/plugins/alerting/server/saved_objects/migrations.ts
@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import { isRuleType, ruleTypeMappings } from '@kbn/securitysolution-rules';
 import { isString } from 'lodash/fp';
 import {
  LogMeta,
@@ -52,7 +53,8 @@ export const isAnyActionSupportIncidents = (doc: SavedObjectUnsanitizedDoc<RawAl
  SUPPORT_INCIDENTS_ACTION_TYPES.includes(action.actionTypeId)
  );
 
-export const isSecuritySolutionRule = (doc: SavedObjectUnsanitizedDoc<RawAlert>): boolean =>
+// Deprecated in 8.0
+export const isSiemSignalsRuleType = (doc: SavedObjectUnsanitizedDoc<RawAlert>): boolean =>
  doc.attributes.alertTypeId === 'siem.signals';
 
 /**
@@ -96,19 +98,19 @@ export function getMigrations(
 
  const migrationSecurityRules713 = createEsoMigration(
  encryptedSavedObjects,
- (doc): doc is SavedObjectUnsanitizedDoc<RawAlert> => isSecuritySolutionRule(doc),
+ (doc): doc is SavedObjectUnsanitizedDoc<RawAlert> => isSiemSignalsRuleType(doc),
  pipeMigrations(removeNullsFromSecurityRules)
  );
 
  const migrationSecurityRules714 = createEsoMigration(
  encryptedSavedObjects,
- (doc): doc is SavedObjectUnsanitizedDoc<RawAlert> => isSecuritySolutionRule(doc),
+ (doc): doc is SavedObjectUnsanitizedDoc<RawAlert> => isSiemSignalsRuleType(doc),
  pipeMigrations(removeNullAuthorFromSecurityRules)
  );
 
  const migrationSecurityRules715 = createEsoMigration(
  encryptedSavedObjects,
- (doc): doc is SavedObjectUnsanitizedDoc<RawAlert> => isSecuritySolutionRule(doc),
+ (doc): doc is SavedObjectUnsanitizedDoc<RawAlert> => isSiemSignalsRuleType(doc),
  pipeMigrations(addExceptionListsToReferences)
  );
 
@@ -126,7 +128,7 @@ export function getMigrations(
  const migrationRules800 = createEsoMigration(
  encryptedSavedObjects,
  (doc: SavedObjectUnsanitizedDoc<RawAlert>): doc is SavedObjectUnsanitizedDoc<RawAlert> => true,
- (doc) => doc // no-op
+ pipeMigrations(addRACRuleTypes)
  );
 
  return {
@@ -647,6 +649,25 @@ function setLegacyId(
  };
 }
 
+function addRACRuleTypes(
+ doc: SavedObjectUnsanitizedDoc<RawAlert>
+): SavedObjectUnsanitizedDoc<RawAlert> {
+ const ruleType = doc.attributes.params.type;
+ return isSiemSignalsRuleType(doc) && isRuleType(ruleType)
+ ? {
+ ...doc,
+ attributes: {
+ ...doc.attributes,
+ alertTypeId: ruleTypeMappings[ruleType],
+ params: {
+ ...doc.attributes.params,
+ outputIndex: '',
+ },
+ },
+ }
+ : doc;
+}
+
 function getRemovePreconfiguredConnectorsFromReferencesFn(
  isPreconfigured: (connectorId: string) => boolean
 ) {