diff --git a/.github/ISSUE_TEMPLATE/Bug_report_security_solution.md b/.github/ISSUE_TEMPLATE/Bug_report_security_solution.md index 059e1d267c2862..7a0514bca621dd 100644 --- a/.github/ISSUE_TEMPLATE/Bug_report_security_solution.md +++ b/.github/ISSUE_TEMPLATE/Bug_report_security_solution.md @@ -1,38 +1,38 @@ ---- -name: Bug report for Security Solution -about: Help us identify bugs in Elastic Security, SIEM, and Endpoint so we can fix them! -title: '[Security Solution]' -labels: 'Team: SecuritySolution' ---- - -**Describe the bug:** - -**Kibana/Elasticsearch Stack version:** - -**Server OS version:** - -**Browser and Browser OS versions:** - -**Elastic Endpoint version:** - -**Original install method (e.g. download page, yum, from source, etc.):** - -**Functional Area (e.g. Endpoint management, timelines, resolver, etc.):** - -**Steps to reproduce:** - -1. -2. -3. - -**Current behavior:** - -**Expected behavior:** - -**Screenshots (if relevant):** - -**Errors in browser console (if relevant):** - -**Provide logs and/or server output (if relevant):** - -**Any additional context (logs, chat logs, magical formulas, etc.):** +--- +name: Bug report for Security Solution +about: Help us identify bugs in Elastic Security, SIEM, and Endpoint so we can fix them! +title: '[Security Solution]' +labels: 'bug, Team: SecuritySolution' +--- + +**Describe the bug:** + +**Kibana/Elasticsearch Stack version:** + +**Server OS version:** + +**Browser and Browser OS versions:** + +**Elastic Endpoint version:** + +**Original install method (e.g. download page, yum, from source, etc.):** + +**Functional Area (e.g. Endpoint management, timelines, resolver, etc.):** + +**Steps to reproduce:** + +1. +2. +3. + +**Current behavior:** + +**Expected behavior:** + +**Screenshots (if relevant):** + +**Errors in browser console (if relevant):** + +**Provide logs and/or server output (if relevant):** + +**Any additional context (logs, chat logs, magical formulas, etc.):** diff --git a/.github/ISSUE_TEMPLATE/v8_breaking_change.md b/.github/ISSUE_TEMPLATE/v8_breaking_change.md index 99f779c288f5b9..c91b937586a09f 100644 --- a/.github/ISSUE_TEMPLATE/v8_breaking_change.md +++ b/.github/ISSUE_TEMPLATE/v8_breaking_change.md @@ -2,7 +2,7 @@ name: 8.0 Breaking change about: Breaking changes from 7.x -> 8.0 title: "[Breaking change]" -labels: Team:Elasticsearch UI, Feature:Upgrade Assistant +labels: Team:Elasticsearch UI, Feature:Upgrade Assistant, Breaking Change assignees: '' --- @@ -11,15 +11,16 @@ assignees: '' **Which release will ship the breaking change?** - +8.0 **Describe the change. How will it manifest to users?** -**What percentage of users will be affected?** +**How many users will be affected?** - + + -**What can users to do to address the change manually?** +**What can users do to address the change manually?** diff --git a/.telemetryrc.json b/.telemetryrc.json index 3d1b0df1d8f93b..0f1530c6225d6a 100644 --- a/.telemetryrc.json +++ b/.telemetryrc.json @@ -7,5 +7,10 @@ "src/plugins/testbed/", "src/plugins/kibana_utils/" ] + }, + { + "output": "src/plugins/telemetry/schema/legacy_plugins.json", + "root": "src/legacy/server/", + "exclude": [] } ] diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.painlesserror._constructor_.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.painlesserror._constructor_.md index f8966572afbb66..051414eac75856 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.painlesserror._constructor_.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.painlesserror._constructor_.md @@ -9,13 +9,13 @@ Constructs a new instance of the `PainlessError` class Signature: ```typescript -constructor(err: EsError, request: IKibanaSearchRequest); +constructor(err: IEsError, request: IKibanaSearchRequest); ``` ## Parameters | Parameter | Type | Description | | --- | --- | --- | -| err | EsError | | +| err | IEsError | | | request | IKibanaSearchRequest | | diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.painlesserror.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.painlesserror.md index 306211cd602599..6ab32f3fb1dfad 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.painlesserror.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.painlesserror.md @@ -7,7 +7,7 @@ Signature: ```typescript -export declare class PainlessError extends KbnError +export declare class PainlessError extends EsError ``` ## Constructors diff --git a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpatternsfetcher._constructor_.md b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpatternsfetcher._constructor_.md index d36ebd0745e8d0..214c795fda9d1a 100644 --- a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpatternsfetcher._constructor_.md +++ b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpatternsfetcher._constructor_.md @@ -9,12 +9,13 @@ Constructs a new instance of the `IndexPatternsFetcher` class Signature: ```typescript -constructor(callDataCluster: LegacyAPICaller); +constructor(elasticsearchClient: ElasticsearchClient, allowNoIndices?: boolean); ``` ## Parameters | Parameter | Type | Description | | --- | --- | --- | -| callDataCluster | LegacyAPICaller | | +| elasticsearchClient | ElasticsearchClient | | +| allowNoIndices | boolean | | diff --git a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpatternsfetcher.getfieldsforwildcard.md b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpatternsfetcher.getfieldsforwildcard.md index 52382372d6d967..addd29916d81df 100644 --- a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpatternsfetcher.getfieldsforwildcard.md +++ b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpatternsfetcher.getfieldsforwildcard.md @@ -13,7 +13,7 @@ getFieldsForWildcard(options: { pattern: string | string[]; metaFields?: string[]; fieldCapsOptions?: { - allowNoIndices: boolean; + allow_no_indices: boolean; }; }): Promise; ``` @@ -22,7 +22,7 @@ getFieldsForWildcard(options: { | Parameter | Type | Description | | --- | --- | --- | -| options | {
pattern: string | string[];
metaFields?: string[];
fieldCapsOptions?: {
allowNoIndices: boolean;
};
} | | +| options | {
pattern: string | string[];
metaFields?: string[];
fieldCapsOptions?: {
allow_no_indices: boolean;
};
} | | Returns: diff --git a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpatternsfetcher.md b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpatternsfetcher.md index f71a702f3381da..3ba3c862bf16a9 100644 --- a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpatternsfetcher.md +++ b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.indexpatternsfetcher.md @@ -14,7 +14,7 @@ export declare class IndexPatternsFetcher | Constructor | Modifiers | Description | | --- | --- | --- | -| [(constructor)(callDataCluster)](./kibana-plugin-plugins-data-server.indexpatternsfetcher._constructor_.md) | | Constructs a new instance of the IndexPatternsFetcher class | +| [(constructor)(elasticsearchClient, allowNoIndices)](./kibana-plugin-plugins-data-server.indexpatternsfetcher._constructor_.md) | | Constructs a new instance of the IndexPatternsFetcher class | ## Methods diff --git a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.execution._constructor_.md b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.execution._constructor_.md index 1d0c9f99169db5..14a0f8818e903d 100644 --- a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.execution._constructor_.md +++ b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.execution._constructor_.md @@ -9,12 +9,12 @@ Constructs a new instance of the `Execution` class Signature: ```typescript -constructor(params: ExecutionParams); +constructor(execution: ExecutionParams); ``` ## Parameters | Parameter | Type | Description | | --- | --- | --- | -| params | ExecutionParams<ExtraContext> | | +| execution | ExecutionParams | | diff --git a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.execution.context.md b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.execution.context.md index 732fe94d656179..e884db46563b52 100644 --- a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.execution.context.md +++ b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.execution.context.md @@ -9,5 +9,5 @@ Execution context - object that allows to do side-effects. Context is passed to Signature: ```typescript -readonly context: ExecutionContext & ExtraContext; +readonly context: ExecutionContext; ``` diff --git a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.execution.contract.md b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.execution.contract.md index fa03297ea22a7d..383e9ee3e81b8c 100644 --- a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.execution.contract.md +++ b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.execution.contract.md @@ -9,5 +9,5 @@ Contract is a public representation of `Execution` instances. Contract we can re Signature: ```typescript -readonly contract: ExecutionContract; +readonly contract: ExecutionContract; ``` diff --git a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.execution.params.md b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.execution.execution.md similarity index 67% rename from docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.execution.params.md rename to docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.execution.execution.md index cd90bf6adab472..eebb5cf5440d52 100644 --- a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.execution.params.md +++ b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.execution.execution.md @@ -1,11 +1,11 @@ -[Home](./index.md) > [kibana-plugin-plugins-expressions-public](./kibana-plugin-plugins-expressions-public.md) > [Execution](./kibana-plugin-plugins-expressions-public.execution.md) > [params](./kibana-plugin-plugins-expressions-public.execution.params.md) +[Home](./index.md) > [kibana-plugin-plugins-expressions-public](./kibana-plugin-plugins-expressions-public.md) > [Execution](./kibana-plugin-plugins-expressions-public.execution.md) > [execution](./kibana-plugin-plugins-expressions-public.execution.execution.md) -## Execution.params property +## Execution.execution property Signature: ```typescript -readonly params: ExecutionParams; +readonly execution: ExecutionParams; ``` diff --git a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.execution.interpret.md b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.execution.interpret.md index 31f38b70698122..24dee04861b4ee 100644 --- a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.execution.interpret.md +++ b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.execution.interpret.md @@ -7,7 +7,7 @@ Signature: ```typescript -interpret(ast: ExpressionAstNode, input: T, options?: ExpressionExecOptions): Promise; +interpret(ast: ExpressionAstNode, input: T): Promise; ``` ## Parameters @@ -16,7 +16,6 @@ interpret(ast: ExpressionAstNode, input: T, options?: ExpressionExecOptions): | --- | --- | --- | | ast | ExpressionAstNode | | | input | T | | -| options | ExpressionExecOptions | | Returns: diff --git a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.execution.md b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.execution.md index 4d227e6ab85b83..56b14e005adfb8 100644 --- a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.execution.md +++ b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.execution.md @@ -7,25 +7,25 @@ Signature: ```typescript -export declare class Execution = Record, Input = unknown, Output = unknown, InspectorAdapters extends Adapters = ExtraContext['inspectorAdapters'] extends object ? ExtraContext['inspectorAdapters'] : DefaultInspectorAdapters> +export declare class Execution ``` ## Constructors | Constructor | Modifiers | Description | | --- | --- | --- | -| [(constructor)(params)](./kibana-plugin-plugins-expressions-public.execution._constructor_.md) | | Constructs a new instance of the Execution class | +| [(constructor)(execution)](./kibana-plugin-plugins-expressions-public.execution._constructor_.md) | | Constructs a new instance of the Execution class | ## Properties | Property | Modifiers | Type | Description | | --- | --- | --- | --- | -| [context](./kibana-plugin-plugins-expressions-public.execution.context.md) | | ExecutionContext<Input, InspectorAdapters> & ExtraContext | Execution context - object that allows to do side-effects. Context is passed to every function. | -| [contract](./kibana-plugin-plugins-expressions-public.execution.contract.md) | | ExecutionContract<ExtraContext, Input, Output, InspectorAdapters> | Contract is a public representation of Execution instances. Contract we can return to other plugins for their consumption. | +| [context](./kibana-plugin-plugins-expressions-public.execution.context.md) | | ExecutionContext<InspectorAdapters> | Execution context - object that allows to do side-effects. Context is passed to every function. | +| [contract](./kibana-plugin-plugins-expressions-public.execution.contract.md) | | ExecutionContract<Input, Output, InspectorAdapters> | Contract is a public representation of Execution instances. Contract we can return to other plugins for their consumption. | +| [execution](./kibana-plugin-plugins-expressions-public.execution.execution.md) | | ExecutionParams | | | [expression](./kibana-plugin-plugins-expressions-public.execution.expression.md) | | string | | | [input](./kibana-plugin-plugins-expressions-public.execution.input.md) | | Input | Initial input of the execution.N.B. It is initialized to null rather than undefined for legacy reasons, because in legacy interpreter it was set to null by default. | | [inspectorAdapters](./kibana-plugin-plugins-expressions-public.execution.inspectoradapters.md) | | InspectorAdapters | | -| [params](./kibana-plugin-plugins-expressions-public.execution.params.md) | | ExecutionParams<ExtraContext> | | | [result](./kibana-plugin-plugins-expressions-public.execution.result.md) | | Promise<Output | ExpressionValueError> | | | [state](./kibana-plugin-plugins-expressions-public.execution.state.md) | | ExecutionContainer<Output | ExpressionValueError> | Dynamic state of the execution. | @@ -35,7 +35,7 @@ export declare class Execution = Re | --- | --- | --- | | [cancel()](./kibana-plugin-plugins-expressions-public.execution.cancel.md) | | Stop execution of expression. | | [cast(value, toTypeNames)](./kibana-plugin-plugins-expressions-public.execution.cast.md) | | | -| [interpret(ast, input, options)](./kibana-plugin-plugins-expressions-public.execution.interpret.md) | | | +| [interpret(ast, input)](./kibana-plugin-plugins-expressions-public.execution.interpret.md) | | | | [invokeChain(chainArr, input)](./kibana-plugin-plugins-expressions-public.execution.invokechain.md) | | | | [invokeFunction(fn, input, args)](./kibana-plugin-plugins-expressions-public.execution.invokefunction.md) | | | | [resolveArgs(fnDef, input, argAsts)](./kibana-plugin-plugins-expressions-public.execution.resolveargs.md) | | | diff --git a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executioncontext.getinitialinput.md b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executioncontext.getsearchcontext.md similarity index 55% rename from docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executioncontext.getinitialinput.md rename to docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executioncontext.getsearchcontext.md index 460b1622c6fbd5..471e18ee6a7ebd 100644 --- a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executioncontext.getinitialinput.md +++ b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executioncontext.getsearchcontext.md @@ -1,13 +1,13 @@ -[Home](./index.md) > [kibana-plugin-plugins-expressions-public](./kibana-plugin-plugins-expressions-public.md) > [ExecutionContext](./kibana-plugin-plugins-expressions-public.executioncontext.md) > [getInitialInput](./kibana-plugin-plugins-expressions-public.executioncontext.getinitialinput.md) +[Home](./index.md) > [kibana-plugin-plugins-expressions-public](./kibana-plugin-plugins-expressions-public.md) > [ExecutionContext](./kibana-plugin-plugins-expressions-public.executioncontext.md) > [getSearchContext](./kibana-plugin-plugins-expressions-public.executioncontext.getsearchcontext.md) -## ExecutionContext.getInitialInput property +## ExecutionContext.getSearchContext property -Get initial input with which execution started. +Get search context of the expression. Signature: ```typescript -getInitialInput: () => Input; +getSearchContext: () => ExecutionContextSearch; ``` diff --git a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executioncontext.search.md b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executioncontext.getsearchsessionid.md similarity index 63% rename from docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executioncontext.search.md rename to docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executioncontext.getsearchsessionid.md index 05501a475cbd40..107ae16dc89012 100644 --- a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executioncontext.search.md +++ b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executioncontext.getsearchsessionid.md @@ -1,13 +1,13 @@ -[Home](./index.md) > [kibana-plugin-plugins-expressions-public](./kibana-plugin-plugins-expressions-public.md) > [ExecutionContext](./kibana-plugin-plugins-expressions-public.executioncontext.md) > [search](./kibana-plugin-plugins-expressions-public.executioncontext.search.md) +[Home](./index.md) > [kibana-plugin-plugins-expressions-public](./kibana-plugin-plugins-expressions-public.md) > [ExecutionContext](./kibana-plugin-plugins-expressions-public.executioncontext.md) > [getSearchSessionId](./kibana-plugin-plugins-expressions-public.executioncontext.getsearchsessionid.md) -## ExecutionContext.search property +## ExecutionContext.getSearchSessionId property Search context in which expression should operate. Signature: ```typescript -search?: ExecutionContextSearch; +getSearchSessionId: () => string | undefined; ``` diff --git a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executioncontext.md b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executioncontext.md index 786e94455c600e..2a1a78b8fcb1a2 100644 --- a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executioncontext.md +++ b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executioncontext.md @@ -9,7 +9,7 @@ Signature: ```typescript -export interface ExecutionContext +export interface ExecutionContext ``` ## Properties @@ -17,10 +17,10 @@ export interface ExecutionContextAbortSignal | Adds ability to abort current execution. | -| [getInitialInput](./kibana-plugin-plugins-expressions-public.executioncontext.getinitialinput.md) | () => Input | Get initial input with which execution started. | | [getSavedObject](./kibana-plugin-plugins-expressions-public.executioncontext.getsavedobject.md) | <T extends SavedObjectAttributes = SavedObjectAttributes>(type: string, id: string) => Promise<SavedObject<T>> | Allows to fetch saved objects from ElasticSearch. In browser getSavedObject function is provided automatically by the Expressions plugin. On the server the caller of the expression has to provide this context function. The reason is because on the browser we always know the user who tries to fetch a saved object, thus saved object client is scoped automatically to that user. However, on the server we can scope that saved object client to any user, or even not scope it at all and execute it as an "internal" user. | +| [getSearchContext](./kibana-plugin-plugins-expressions-public.executioncontext.getsearchcontext.md) | () => ExecutionContextSearch | Get search context of the expression. | +| [getSearchSessionId](./kibana-plugin-plugins-expressions-public.executioncontext.getsearchsessionid.md) | () => string | undefined | Search context in which expression should operate. | | [inspectorAdapters](./kibana-plugin-plugins-expressions-public.executioncontext.inspectoradapters.md) | InspectorAdapters | Adapters for inspector plugin. | -| [search](./kibana-plugin-plugins-expressions-public.executioncontext.search.md) | ExecutionContextSearch | Search context in which expression should operate. | | [types](./kibana-plugin-plugins-expressions-public.executioncontext.types.md) | Record<string, ExpressionType> | A map of available expression types. | | [variables](./kibana-plugin-plugins-expressions-public.executioncontext.variables.md) | Record<string, unknown> | Context variables that can be consumed using var and var_set functions. | diff --git a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executioncontract._constructor_.md b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executioncontract._constructor_.md index 89a99ef2f8ef83..ee8b113881a05a 100644 --- a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executioncontract._constructor_.md +++ b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executioncontract._constructor_.md @@ -9,12 +9,12 @@ Constructs a new instance of the `ExecutionContract` class Signature: ```typescript -constructor(execution: Execution); +constructor(execution: Execution); ``` ## Parameters | Parameter | Type | Description | | --- | --- | --- | -| execution | Execution<ExtraContext, Input, Output, InspectorAdapters> | | +| execution | Execution<Input, Output, InspectorAdapters> | | diff --git a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executioncontract.execution.md b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executioncontract.execution.md index f7911250488f22..aa058c71c12dff 100644 --- a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executioncontract.execution.md +++ b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executioncontract.execution.md @@ -7,5 +7,5 @@ Signature: ```typescript -protected readonly execution: Execution; +protected readonly execution: Execution; ``` diff --git a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executioncontract.md b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executioncontract.md index d05620eace208c..f2c050bbfe0ba2 100644 --- a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executioncontract.md +++ b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executioncontract.md @@ -9,7 +9,7 @@ Signature: ```typescript -export declare class ExecutionContract = Record, Input = unknown, Output = unknown, InspectorAdapters = unknown> +export declare class ExecutionContract ``` ## Constructors @@ -23,7 +23,7 @@ export declare class ExecutionContract() => void | Cancel the execution of the expression. This will set abort signal (available in execution context) to aborted state, letting expression functions to stop their execution. | -| [execution](./kibana-plugin-plugins-expressions-public.executioncontract.execution.md) | | Execution<ExtraContext, Input, Output, InspectorAdapters> | | +| [execution](./kibana-plugin-plugins-expressions-public.executioncontract.execution.md) | | Execution<Input, Output, InspectorAdapters> | | | [getAst](./kibana-plugin-plugins-expressions-public.executioncontract.getast.md) | | () => ExpressionAstExpression | Get AST used to execute the expression. | | [getData](./kibana-plugin-plugins-expressions-public.executioncontract.getdata.md) | | () => Promise<Output | ExpressionValueError> | Returns the final output of expression, if any error happens still wraps that error into ExpressionValueError type and returns that. This function never throws. | | [getExpression](./kibana-plugin-plugins-expressions-public.executioncontract.getexpression.md) | | () => string | Get string representation of the expression. Returns the original string if execution was started from a string. If execution was started from an AST this method returns a string generated from AST. | diff --git a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executionparams.debug.md b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executionparams.debug.md deleted file mode 100644 index 61ec72465f55ed..00000000000000 --- a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executionparams.debug.md +++ /dev/null @@ -1,13 +0,0 @@ - - -[Home](./index.md) > [kibana-plugin-plugins-expressions-public](./kibana-plugin-plugins-expressions-public.md) > [ExecutionParams](./kibana-plugin-plugins-expressions-public.executionparams.md) > [debug](./kibana-plugin-plugins-expressions-public.executionparams.debug.md) - -## ExecutionParams.debug property - -Whether to execute expression in \*debug mode\*. In \*debug mode\* inputs and outputs as well as all resolved arguments and time it took to execute each function are saved and are available for introspection. - -Signature: - -```typescript -debug?: boolean; -``` diff --git a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executionparams.md b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executionparams.md index e39dc231fbf969..6e5d70c61ead6f 100644 --- a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executionparams.md +++ b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executionparams.md @@ -7,7 +7,7 @@ Signature: ```typescript -export interface ExecutionParams = Record> +export interface ExecutionParams ``` ## Properties @@ -15,8 +15,7 @@ export interface ExecutionParams = | Property | Type | Description | | --- | --- | --- | | [ast](./kibana-plugin-plugins-expressions-public.executionparams.ast.md) | ExpressionAstExpression | | -| [context](./kibana-plugin-plugins-expressions-public.executionparams.context.md) | ExtraContext | | -| [debug](./kibana-plugin-plugins-expressions-public.executionparams.debug.md) | boolean | Whether to execute expression in \*debug mode\*. In \*debug mode\* inputs and outputs as well as all resolved arguments and time it took to execute each function are saved and are available for introspection. | | [executor](./kibana-plugin-plugins-expressions-public.executionparams.executor.md) | Executor<any> | | | [expression](./kibana-plugin-plugins-expressions-public.executionparams.expression.md) | string | | +| [params](./kibana-plugin-plugins-expressions-public.executionparams.params.md) | ExpressionExecutionParams | | diff --git a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executionparams.context.md b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executionparams.params.md similarity index 65% rename from docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executionparams.context.md rename to docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executionparams.params.md index b6013162ef2ae1..0dbe87bfda79e4 100644 --- a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executionparams.context.md +++ b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executionparams.params.md @@ -1,11 +1,11 @@ -[Home](./index.md) > [kibana-plugin-plugins-expressions-public](./kibana-plugin-plugins-expressions-public.md) > [ExecutionParams](./kibana-plugin-plugins-expressions-public.executionparams.md) > [context](./kibana-plugin-plugins-expressions-public.executionparams.context.md) +[Home](./index.md) > [kibana-plugin-plugins-expressions-public](./kibana-plugin-plugins-expressions-public.md) > [ExecutionParams](./kibana-plugin-plugins-expressions-public.executionparams.md) > [params](./kibana-plugin-plugins-expressions-public.executionparams.params.md) -## ExecutionParams.context property +## ExecutionParams.params property Signature: ```typescript -context?: ExtraContext; +params: ExpressionExecutionParams; ``` diff --git a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executor.createexecution.md b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executor.createexecution.md index e6765064d4a27d..2832ba92262f26 100644 --- a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executor.createexecution.md +++ b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executor.createexecution.md @@ -7,7 +7,7 @@ Signature: ```typescript -createExecution = Record, Input = unknown, Output = unknown>(ast: string | ExpressionAstExpression, context?: ExtraContext, { debug }?: ExpressionExecOptions): Execution; +createExecution(ast: string | ExpressionAstExpression, params?: ExpressionExecutionParams): Execution; ``` ## Parameters @@ -15,10 +15,9 @@ createExecution = Recordstring | ExpressionAstExpression | | -| context | ExtraContext | | -| { debug } | ExpressionExecOptions | | +| params | ExpressionExecutionParams | | Returns: -`Execution` +`Execution` diff --git a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executor.md b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executor.md index 013624f30b45a7..aefd04112dc1ca 100644 --- a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executor.md +++ b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.executor.md @@ -29,7 +29,7 @@ export declare class Executor = Recordstatic | | | [extendContext(extraContext)](./kibana-plugin-plugins-expressions-public.executor.extendcontext.md) | | | | [extract(ast)](./kibana-plugin-plugins-expressions-public.executor.extract.md) | | | @@ -41,6 +41,6 @@ export declare class Executor = RecordSignature: ```typescript -run = Record>(ast: string | ExpressionAstExpression, input: Input, context?: ExtraContext, options?: ExpressionExecOptions): Promise; +run(ast: string | ExpressionAstExpression, input: Input, params?: ExpressionExecutionParams): Promise; ``` ## Parameters @@ -18,8 +18,7 @@ run = Recordstring | ExpressionAstExpression | | | input | Input | | -| context | ExtraContext | | -| options | ExpressionExecOptions | | +| params | ExpressionExecutionParams | | Returns: diff --git a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.expressionsservicestart.execute.md b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.expressionsservicestart.execute.md index 18b856b946da47..043d3472228a25 100644 --- a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.expressionsservicestart.execute.md +++ b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.expressionsservicestart.execute.md @@ -9,5 +9,5 @@ Starts expression execution and immediately returns `ExecutionContract` instance Signature: ```typescript -execute: = Record>(ast: string | ExpressionAstExpression, input: Input, context?: ExtraContext, options?: ExpressionExecOptions) => ExecutionContract; +execute: (ast: string | ExpressionAstExpression, input: Input, params?: ExpressionExecutionParams) => ExecutionContract; ``` diff --git a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.expressionsservicestart.md b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.expressionsservicestart.md index def572abead225..6b678fc4fbc264 100644 --- a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.expressionsservicestart.md +++ b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.expressionsservicestart.md @@ -16,12 +16,12 @@ export interface ExpressionsServiceStart | Property | Type | Description | | --- | --- | --- | -| [execute](./kibana-plugin-plugins-expressions-public.expressionsservicestart.execute.md) | <Input = unknown, Output = unknown, ExtraContext extends Record<string, unknown> = Record<string, unknown>>(ast: string | ExpressionAstExpression, input: Input, context?: ExtraContext, options?: ExpressionExecOptions) => ExecutionContract<ExtraContext, Input, Output> | Starts expression execution and immediately returns ExecutionContract instance that tracks the progress of the execution and can be used to interact with the execution. | +| [execute](./kibana-plugin-plugins-expressions-public.expressionsservicestart.execute.md) | <Input = unknown, Output = unknown>(ast: string | ExpressionAstExpression, input: Input, params?: ExpressionExecutionParams) => ExecutionContract<Input, Output> | Starts expression execution and immediately returns ExecutionContract instance that tracks the progress of the execution and can be used to interact with the execution. | | [fork](./kibana-plugin-plugins-expressions-public.expressionsservicestart.fork.md) | () => ExpressionsService | Create a new instance of ExpressionsService. The new instance inherits all state of the original ExpressionsService, including all expression types, expression functions and context. Also, all new types and functions registered in the original services AFTER the forking event will be available in the forked instance. However, all new types and functions registered in the forked instances will NOT be available to the original service. | | [getFunction](./kibana-plugin-plugins-expressions-public.expressionsservicestart.getfunction.md) | (name: string) => ReturnType<Executor['getFunction']> | Get a registered ExpressionFunction by its name, which was registered using the registerFunction method. The returned ExpressionFunction instance is an internal representation of the function in Expressions service - do not mutate that object. | | [getRenderer](./kibana-plugin-plugins-expressions-public.expressionsservicestart.getrenderer.md) | (name: string) => ReturnType<ExpressionRendererRegistry['get']> | Get a registered ExpressionRenderer by its name, which was registered using the registerRenderer method. The returned ExpressionRenderer instance is an internal representation of the renderer in Expressions service - do not mutate that object. | | [getType](./kibana-plugin-plugins-expressions-public.expressionsservicestart.gettype.md) | (name: string) => ReturnType<Executor['getType']> | Get a registered ExpressionType by its name, which was registered using the registerType method. The returned ExpressionType instance is an internal representation of the type in Expressions service - do not mutate that object. | -| [run](./kibana-plugin-plugins-expressions-public.expressionsservicestart.run.md) | <Input, Output, ExtraContext extends Record<string, unknown> = Record<string, unknown>>(ast: string | ExpressionAstExpression, input: Input, context?: ExtraContext, options?: ExpressionExecOptions) => Promise<Output> | Executes expression string or a parsed expression AST and immediately returns the result.Below example will execute sleep 100 | clog expression with 123 initial input to the first function. +| [run](./kibana-plugin-plugins-expressions-public.expressionsservicestart.run.md) | <Input, Output>(ast: string | ExpressionAstExpression, input: Input, params?: ExpressionExecutionParams) => Promise<Output> | Executes expression string or a parsed expression AST and immediately returns the result.Below example will execute sleep 100 | clog expression with 123 initial input to the first function. ```ts expressions.run('sleep 100 | clog', 123); diff --git a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.expressionsservicestart.run.md b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.expressionsservicestart.run.md index d717af51a00fa9..9efca0011174cc 100644 --- a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.expressionsservicestart.run.md +++ b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.expressionsservicestart.run.md @@ -24,5 +24,5 @@ expressions.run('...', null, { elasticsearchClient }); Signature: ```typescript -run: = Record>(ast: string | ExpressionAstExpression, input: Input, context?: ExtraContext, options?: ExpressionExecOptions) => Promise; +run: (ast: string | ExpressionAstExpression, input: Input, params?: ExpressionExecutionParams) => Promise; ``` diff --git a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.iexpressionloaderparams.md b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.iexpressionloaderparams.md index d6e02350bae3f6..e2ad6215e25d09 100644 --- a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.iexpressionloaderparams.md +++ b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.iexpressionloaderparams.md @@ -22,6 +22,7 @@ export interface IExpressionLoaderParams | [inspectorAdapters](./kibana-plugin-plugins-expressions-public.iexpressionloaderparams.inspectoradapters.md) | Adapters | | | [onRenderError](./kibana-plugin-plugins-expressions-public.iexpressionloaderparams.onrendererror.md) | RenderErrorHandlerFnType | | | [searchContext](./kibana-plugin-plugins-expressions-public.iexpressionloaderparams.searchcontext.md) | ExecutionContextSearch | | +| [searchSessionId](./kibana-plugin-plugins-expressions-public.iexpressionloaderparams.searchsessionid.md) | string | | | [uiState](./kibana-plugin-plugins-expressions-public.iexpressionloaderparams.uistate.md) | unknown | | | [variables](./kibana-plugin-plugins-expressions-public.iexpressionloaderparams.variables.md) | Record<string, any> | | diff --git a/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.iexpressionloaderparams.searchsessionid.md b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.iexpressionloaderparams.searchsessionid.md new file mode 100644 index 00000000000000..bb021b003f0d3e --- /dev/null +++ b/docs/development/plugins/expressions/public/kibana-plugin-plugins-expressions-public.iexpressionloaderparams.searchsessionid.md @@ -0,0 +1,11 @@ + + +[Home](./index.md) > [kibana-plugin-plugins-expressions-public](./kibana-plugin-plugins-expressions-public.md) > [IExpressionLoaderParams](./kibana-plugin-plugins-expressions-public.iexpressionloaderparams.md) > [searchSessionId](./kibana-plugin-plugins-expressions-public.iexpressionloaderparams.searchsessionid.md) + +## IExpressionLoaderParams.searchSessionId property + +Signature: + +```typescript +searchSessionId?: string; +``` diff --git a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.execution._constructor_.md b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.execution._constructor_.md index 75f4cc4c2a0173..f24aae8603b7d7 100644 --- a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.execution._constructor_.md +++ b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.execution._constructor_.md @@ -9,12 +9,12 @@ Constructs a new instance of the `Execution` class Signature: ```typescript -constructor(params: ExecutionParams); +constructor(execution: ExecutionParams); ``` ## Parameters | Parameter | Type | Description | | --- | --- | --- | -| params | ExecutionParams<ExtraContext> | | +| execution | ExecutionParams | | diff --git a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.execution.context.md b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.execution.context.md index d1969fb0859b70..65c7bdca0fe5da 100644 --- a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.execution.context.md +++ b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.execution.context.md @@ -9,5 +9,5 @@ Execution context - object that allows to do side-effects. Context is passed to Signature: ```typescript -readonly context: ExecutionContext & ExtraContext; +readonly context: ExecutionContext; ``` diff --git a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.execution.contract.md b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.execution.contract.md index 149b5a7ced9cb7..2fc6a38997f777 100644 --- a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.execution.contract.md +++ b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.execution.contract.md @@ -9,5 +9,5 @@ Contract is a public representation of `Execution` instances. Contract we can re Signature: ```typescript -readonly contract: ExecutionContract; +readonly contract: ExecutionContract; ``` diff --git a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.execution.params.md b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.execution.execution.md similarity index 67% rename from docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.execution.params.md rename to docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.execution.execution.md index 498f9bbfccfa40..acaccdeab73216 100644 --- a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.execution.params.md +++ b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.execution.execution.md @@ -1,11 +1,11 @@ -[Home](./index.md) > [kibana-plugin-plugins-expressions-server](./kibana-plugin-plugins-expressions-server.md) > [Execution](./kibana-plugin-plugins-expressions-server.execution.md) > [params](./kibana-plugin-plugins-expressions-server.execution.params.md) +[Home](./index.md) > [kibana-plugin-plugins-expressions-server](./kibana-plugin-plugins-expressions-server.md) > [Execution](./kibana-plugin-plugins-expressions-server.execution.md) > [execution](./kibana-plugin-plugins-expressions-server.execution.execution.md) -## Execution.params property +## Execution.execution property Signature: ```typescript -readonly params: ExecutionParams; +readonly execution: ExecutionParams; ``` diff --git a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.execution.interpret.md b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.execution.interpret.md index cf59e796e61208..e425bdc70e3491 100644 --- a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.execution.interpret.md +++ b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.execution.interpret.md @@ -7,7 +7,7 @@ Signature: ```typescript -interpret(ast: ExpressionAstNode, input: T, options?: ExpressionExecOptions): Promise; +interpret(ast: ExpressionAstNode, input: T): Promise; ``` ## Parameters @@ -16,7 +16,6 @@ interpret(ast: ExpressionAstNode, input: T, options?: ExpressionExecOptions): | --- | --- | --- | | ast | ExpressionAstNode | | | input | T | | -| options | ExpressionExecOptions | | Returns: diff --git a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.execution.md b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.execution.md index fc663dd1155807..c94ae9bcfe9466 100644 --- a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.execution.md +++ b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.execution.md @@ -7,25 +7,25 @@ Signature: ```typescript -export declare class Execution = Record, Input = unknown, Output = unknown, InspectorAdapters extends Adapters = ExtraContext['inspectorAdapters'] extends object ? ExtraContext['inspectorAdapters'] : DefaultInspectorAdapters> +export declare class Execution ``` ## Constructors | Constructor | Modifiers | Description | | --- | --- | --- | -| [(constructor)(params)](./kibana-plugin-plugins-expressions-server.execution._constructor_.md) | | Constructs a new instance of the Execution class | +| [(constructor)(execution)](./kibana-plugin-plugins-expressions-server.execution._constructor_.md) | | Constructs a new instance of the Execution class | ## Properties | Property | Modifiers | Type | Description | | --- | --- | --- | --- | -| [context](./kibana-plugin-plugins-expressions-server.execution.context.md) | | ExecutionContext<Input, InspectorAdapters> & ExtraContext | Execution context - object that allows to do side-effects. Context is passed to every function. | -| [contract](./kibana-plugin-plugins-expressions-server.execution.contract.md) | | ExecutionContract<ExtraContext, Input, Output, InspectorAdapters> | Contract is a public representation of Execution instances. Contract we can return to other plugins for their consumption. | +| [context](./kibana-plugin-plugins-expressions-server.execution.context.md) | | ExecutionContext<InspectorAdapters> | Execution context - object that allows to do side-effects. Context is passed to every function. | +| [contract](./kibana-plugin-plugins-expressions-server.execution.contract.md) | | ExecutionContract<Input, Output, InspectorAdapters> | Contract is a public representation of Execution instances. Contract we can return to other plugins for their consumption. | +| [execution](./kibana-plugin-plugins-expressions-server.execution.execution.md) | | ExecutionParams | | | [expression](./kibana-plugin-plugins-expressions-server.execution.expression.md) | | string | | | [input](./kibana-plugin-plugins-expressions-server.execution.input.md) | | Input | Initial input of the execution.N.B. It is initialized to null rather than undefined for legacy reasons, because in legacy interpreter it was set to null by default. | | [inspectorAdapters](./kibana-plugin-plugins-expressions-server.execution.inspectoradapters.md) | | InspectorAdapters | | -| [params](./kibana-plugin-plugins-expressions-server.execution.params.md) | | ExecutionParams<ExtraContext> | | | [result](./kibana-plugin-plugins-expressions-server.execution.result.md) | | Promise<Output | ExpressionValueError> | | | [state](./kibana-plugin-plugins-expressions-server.execution.state.md) | | ExecutionContainer<Output | ExpressionValueError> | Dynamic state of the execution. | @@ -35,7 +35,7 @@ export declare class Execution = Re | --- | --- | --- | | [cancel()](./kibana-plugin-plugins-expressions-server.execution.cancel.md) | | Stop execution of expression. | | [cast(value, toTypeNames)](./kibana-plugin-plugins-expressions-server.execution.cast.md) | | | -| [interpret(ast, input, options)](./kibana-plugin-plugins-expressions-server.execution.interpret.md) | | | +| [interpret(ast, input)](./kibana-plugin-plugins-expressions-server.execution.interpret.md) | | | | [invokeChain(chainArr, input)](./kibana-plugin-plugins-expressions-server.execution.invokechain.md) | | | | [invokeFunction(fn, input, args)](./kibana-plugin-plugins-expressions-server.execution.invokefunction.md) | | | | [resolveArgs(fnDef, input, argAsts)](./kibana-plugin-plugins-expressions-server.execution.resolveargs.md) | | | diff --git a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executioncontext.getinitialinput.md b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executioncontext.getsearchcontext.md similarity index 55% rename from docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executioncontext.getinitialinput.md rename to docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executioncontext.getsearchcontext.md index b5f9b91e1c7b71..783841fbafa923 100644 --- a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executioncontext.getinitialinput.md +++ b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executioncontext.getsearchcontext.md @@ -1,13 +1,13 @@ -[Home](./index.md) > [kibana-plugin-plugins-expressions-server](./kibana-plugin-plugins-expressions-server.md) > [ExecutionContext](./kibana-plugin-plugins-expressions-server.executioncontext.md) > [getInitialInput](./kibana-plugin-plugins-expressions-server.executioncontext.getinitialinput.md) +[Home](./index.md) > [kibana-plugin-plugins-expressions-server](./kibana-plugin-plugins-expressions-server.md) > [ExecutionContext](./kibana-plugin-plugins-expressions-server.executioncontext.md) > [getSearchContext](./kibana-plugin-plugins-expressions-server.executioncontext.getsearchcontext.md) -## ExecutionContext.getInitialInput property +## ExecutionContext.getSearchContext property -Get initial input with which execution started. +Get search context of the expression. Signature: ```typescript -getInitialInput: () => Input; +getSearchContext: () => ExecutionContextSearch; ``` diff --git a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executioncontext.search.md b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executioncontext.getsearchsessionid.md similarity index 63% rename from docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executioncontext.search.md rename to docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executioncontext.getsearchsessionid.md index 641e50696f6e03..a69f0ed32bd514 100644 --- a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executioncontext.search.md +++ b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executioncontext.getsearchsessionid.md @@ -1,13 +1,13 @@ -[Home](./index.md) > [kibana-plugin-plugins-expressions-server](./kibana-plugin-plugins-expressions-server.md) > [ExecutionContext](./kibana-plugin-plugins-expressions-server.executioncontext.md) > [search](./kibana-plugin-plugins-expressions-server.executioncontext.search.md) +[Home](./index.md) > [kibana-plugin-plugins-expressions-server](./kibana-plugin-plugins-expressions-server.md) > [ExecutionContext](./kibana-plugin-plugins-expressions-server.executioncontext.md) > [getSearchSessionId](./kibana-plugin-plugins-expressions-server.executioncontext.getsearchsessionid.md) -## ExecutionContext.search property +## ExecutionContext.getSearchSessionId property Search context in which expression should operate. Signature: ```typescript -search?: ExecutionContextSearch; +getSearchSessionId: () => string | undefined; ``` diff --git a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executioncontext.md b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executioncontext.md index 0128ba934da736..047879fd99255f 100644 --- a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executioncontext.md +++ b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executioncontext.md @@ -9,7 +9,7 @@ Signature: ```typescript -export interface ExecutionContext +export interface ExecutionContext ``` ## Properties @@ -17,10 +17,10 @@ export interface ExecutionContextAbortSignal | Adds ability to abort current execution. | -| [getInitialInput](./kibana-plugin-plugins-expressions-server.executioncontext.getinitialinput.md) | () => Input | Get initial input with which execution started. | | [getSavedObject](./kibana-plugin-plugins-expressions-server.executioncontext.getsavedobject.md) | <T extends SavedObjectAttributes = SavedObjectAttributes>(type: string, id: string) => Promise<SavedObject<T>> | Allows to fetch saved objects from ElasticSearch. In browser getSavedObject function is provided automatically by the Expressions plugin. On the server the caller of the expression has to provide this context function. The reason is because on the browser we always know the user who tries to fetch a saved object, thus saved object client is scoped automatically to that user. However, on the server we can scope that saved object client to any user, or even not scope it at all and execute it as an "internal" user. | +| [getSearchContext](./kibana-plugin-plugins-expressions-server.executioncontext.getsearchcontext.md) | () => ExecutionContextSearch | Get search context of the expression. | +| [getSearchSessionId](./kibana-plugin-plugins-expressions-server.executioncontext.getsearchsessionid.md) | () => string | undefined | Search context in which expression should operate. | | [inspectorAdapters](./kibana-plugin-plugins-expressions-server.executioncontext.inspectoradapters.md) | InspectorAdapters | Adapters for inspector plugin. | -| [search](./kibana-plugin-plugins-expressions-server.executioncontext.search.md) | ExecutionContextSearch | Search context in which expression should operate. | | [types](./kibana-plugin-plugins-expressions-server.executioncontext.types.md) | Record<string, ExpressionType> | A map of available expression types. | | [variables](./kibana-plugin-plugins-expressions-server.executioncontext.variables.md) | Record<string, unknown> | Context variables that can be consumed using var and var_set functions. | diff --git a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executionparams.debug.md b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executionparams.debug.md deleted file mode 100644 index b3631e0aeebe61..00000000000000 --- a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executionparams.debug.md +++ /dev/null @@ -1,13 +0,0 @@ - - -[Home](./index.md) > [kibana-plugin-plugins-expressions-server](./kibana-plugin-plugins-expressions-server.md) > [ExecutionParams](./kibana-plugin-plugins-expressions-server.executionparams.md) > [debug](./kibana-plugin-plugins-expressions-server.executionparams.debug.md) - -## ExecutionParams.debug property - -Whether to execute expression in \*debug mode\*. In \*debug mode\* inputs and outputs as well as all resolved arguments and time it took to execute each function are saved and are available for introspection. - -Signature: - -```typescript -debug?: boolean; -``` diff --git a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executionparams.md b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executionparams.md index a7594bff48c1ae..6a901c91ffff1b 100644 --- a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executionparams.md +++ b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executionparams.md @@ -7,7 +7,7 @@ Signature: ```typescript -export interface ExecutionParams = Record> +export interface ExecutionParams ``` ## Properties @@ -15,8 +15,7 @@ export interface ExecutionParams = | Property | Type | Description | | --- | --- | --- | | [ast](./kibana-plugin-plugins-expressions-server.executionparams.ast.md) | ExpressionAstExpression | | -| [context](./kibana-plugin-plugins-expressions-server.executionparams.context.md) | ExtraContext | | -| [debug](./kibana-plugin-plugins-expressions-server.executionparams.debug.md) | boolean | Whether to execute expression in \*debug mode\*. In \*debug mode\* inputs and outputs as well as all resolved arguments and time it took to execute each function are saved and are available for introspection. | | [executor](./kibana-plugin-plugins-expressions-server.executionparams.executor.md) | Executor<any> | | | [expression](./kibana-plugin-plugins-expressions-server.executionparams.expression.md) | string | | +| [params](./kibana-plugin-plugins-expressions-server.executionparams.params.md) | ExpressionExecutionParams | | diff --git a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executionparams.context.md b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executionparams.params.md similarity index 65% rename from docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executionparams.context.md rename to docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executionparams.params.md index 8b9a210416dd68..fec60af33e0c4a 100644 --- a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executionparams.context.md +++ b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executionparams.params.md @@ -1,11 +1,11 @@ -[Home](./index.md) > [kibana-plugin-plugins-expressions-server](./kibana-plugin-plugins-expressions-server.md) > [ExecutionParams](./kibana-plugin-plugins-expressions-server.executionparams.md) > [context](./kibana-plugin-plugins-expressions-server.executionparams.context.md) +[Home](./index.md) > [kibana-plugin-plugins-expressions-server](./kibana-plugin-plugins-expressions-server.md) > [ExecutionParams](./kibana-plugin-plugins-expressions-server.executionparams.md) > [params](./kibana-plugin-plugins-expressions-server.executionparams.params.md) -## ExecutionParams.context property +## ExecutionParams.params property Signature: ```typescript -context?: ExtraContext; +params: ExpressionExecutionParams; ``` diff --git a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executor.createexecution.md b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executor.createexecution.md index 8ed228d70ff370..34e920a04fd021 100644 --- a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executor.createexecution.md +++ b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executor.createexecution.md @@ -7,7 +7,7 @@ Signature: ```typescript -createExecution = Record, Input = unknown, Output = unknown>(ast: string | ExpressionAstExpression, context?: ExtraContext, { debug }?: ExpressionExecOptions): Execution; +createExecution(ast: string | ExpressionAstExpression, params?: ExpressionExecutionParams): Execution; ``` ## Parameters @@ -15,10 +15,9 @@ createExecution = Recordstring | ExpressionAstExpression | | -| context | ExtraContext | | -| { debug } | ExpressionExecOptions | | +| params | ExpressionExecutionParams | | Returns: -`Execution` +`Execution` diff --git a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executor.md b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executor.md index 46ad60ae07126e..97bb3ac895084d 100644 --- a/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executor.md +++ b/docs/development/plugins/expressions/server/kibana-plugin-plugins-expressions-server.executor.md @@ -29,7 +29,7 @@ export declare class Executor = Recordstatic | | | [extendContext(extraContext)](./kibana-plugin-plugins-expressions-server.executor.extendcontext.md) | | | | [extract(ast)](./kibana-plugin-plugins-expressions-server.executor.extract.md) | | | @@ -41,6 +41,6 @@ export declare class Executor = RecordSignature: ```typescript -run = Record>(ast: string | ExpressionAstExpression, input: Input, context?: ExtraContext, options?: ExpressionExecOptions): Promise; +run(ast: string | ExpressionAstExpression, input: Input, params?: ExpressionExecutionParams): Promise; ``` ## Parameters @@ -18,8 +18,7 @@ run = Recordstring | ExpressionAstExpression | | | input | Input | | -| context | ExtraContext | | -| options | ExpressionExecOptions | | +| params | ExpressionExecutionParams | | Returns: diff --git a/docs/management/advanced-options.asciidoc b/docs/management/advanced-options.asciidoc index 293597685ecc06..dbac6997ff4333 100644 --- a/docs/management/advanced-options.asciidoc +++ b/docs/management/advanced-options.asciidoc @@ -11,6 +11,13 @@ values. . Enter a new value for the setting. . Click *Save changes*. +[float] +=== Required permissions + +The `Advanced Settings` {kib} privilege is required to access *Advanced Settings*. + +To add the privilege, open the menu, then click *Stack Management > Roles*. + [float] [[settings-read-only-access]] diff --git a/docs/management/alerting/alerts-and-actions-intro.asciidoc b/docs/management/alerting/alerts-and-actions-intro.asciidoc index 0c7ca7f1db17db..35a2452e999513 100644 --- a/docs/management/alerting/alerts-and-actions-intro.asciidoc +++ b/docs/management/alerting/alerts-and-actions-intro.asciidoc @@ -24,3 +24,8 @@ The *Alerts and Actions* UI only shows alerts and connectors for the current spa can be managed through the <>. See <> for more information. ============================================================================ + +[float] +=== Required permissions + +Access to alerts and actions is granted based on your privileges to alerting-enabled features. See <> for more information. diff --git a/docs/management/managing-beats.asciidoc b/docs/management/managing-beats.asciidoc index 10c98cca263458..232efb60cadd3e 100644 --- a/docs/management/managing-beats.asciidoc +++ b/docs/management/managing-beats.asciidoc @@ -29,6 +29,13 @@ more information, see https://www.elastic.co/subscriptions and enrollment and configuration process step by step the first time you use the Central Management UI. +[float] +=== Required permissions + +You must have the `beats_admin` role assigned to use **{beats} Central Management** + +To assign the role, open the menu, then click *Stack Management > Users*. + [float] === Enroll {beats} diff --git a/docs/management/managing-fields.asciidoc b/docs/management/managing-fields.asciidoc index 441bce43c7cdf3..3734655edd91b3 100644 --- a/docs/management/managing-fields.asciidoc +++ b/docs/management/managing-fields.asciidoc @@ -7,6 +7,13 @@ the index patterns that retrieve your data from {es}. [role="screenshot"] image::images/management-index-patterns.png[] +[float] +=== Required permissions + +The `Index Pattern Management` {kib} privilege is required to access the *Index patterns* UI. + +To add the privilege, open the menu, then click *Stack Management > Roles*. + [float] === Create an index pattern diff --git a/docs/management/managing-saved-objects.asciidoc b/docs/management/managing-saved-objects.asciidoc index 639be87c540fbf..2e081c09e0e702 100644 --- a/docs/management/managing-saved-objects.asciidoc +++ b/docs/management/managing-saved-objects.asciidoc @@ -10,6 +10,16 @@ To get started, open the main menu, then click *Stack Management > Saved Objects [role="screenshot"] image::images/management-saved-objects.png[Saved Objects] +[float] +=== Required permissions + +The `Saved Objects Management` {kib} privilege is required to access the *Saved Objects* UI. + +To add the privilege, open the menu, then click *Stack Management > Roles*. + +NOTE: +Granting access to Saved Objects Management will authorize users to manage all saved objects in {kib}, including objects that are managed by applications they may not otherwise be authorized to access. + [float] [[managing-saved-objects-view]] diff --git a/docs/spaces/images/edit-space-feature-visibility.png b/docs/spaces/images/edit-space-feature-visibility.png index bc983bde2679ca..3a74c21fe2635f 100644 Binary files a/docs/spaces/images/edit-space-feature-visibility.png and b/docs/spaces/images/edit-space-feature-visibility.png differ diff --git a/docs/spaces/images/edit-space.png b/docs/spaces/images/edit-space.png index 68ffea23c4ac48..c4565a3ba0f4a4 100644 Binary files a/docs/spaces/images/edit-space.png and b/docs/spaces/images/edit-space.png differ diff --git a/docs/spaces/images/space-selector.png b/docs/spaces/images/space-selector.png index 908c5360acd396..b2576cbc9acc4f 100644 Binary files a/docs/spaces/images/space-selector.png and b/docs/spaces/images/space-selector.png differ diff --git a/docs/spaces/images/spaces-roles.png b/docs/spaces/images/spaces-roles.png old mode 100755 new mode 100644 index 2ecbfed6017c4f..407926895daa87 Binary files a/docs/spaces/images/spaces-roles.png and b/docs/spaces/images/spaces-roles.png differ diff --git a/docs/spaces/index.asciidoc b/docs/spaces/index.asciidoc index 1bc781e1dda491..81f39457795035 100644 --- a/docs/spaces/index.asciidoc +++ b/docs/spaces/index.asciidoc @@ -25,6 +25,11 @@ Kibana supports spaces in several ways. You can: * <> * <> +[float] +==== Required permissions + +The `kibana_admin` role or equivilent is required to manage **Spaces**. + [float] [[spaces-managing]] === View, create, and delete spaces diff --git a/docs/user/introduction.asciidoc b/docs/user/introduction.asciidoc index aa5b0ece08db70..91f149d5cdb3cc 100644 --- a/docs/user/introduction.asciidoc +++ b/docs/user/introduction.asciidoc @@ -112,7 +112,7 @@ You can even choose which features to enable within each space. Don’t need Machine learning in your “Executive” space? Simply turn it off. [role="screenshot"] -image::images/intro-spaces.jpg[] +image::images/intro-spaces.png[Space selector screen] You can take this all one step further with Kibana’s security features, and control which users have access to each space. {kib} allows for fine-grained diff --git a/docs/user/introduction/images/intro-spaces.jpg b/docs/user/introduction/images/intro-spaces.jpg deleted file mode 100755 index 7569dfc16b4f79..00000000000000 Binary files a/docs/user/introduction/images/intro-spaces.jpg and /dev/null differ diff --git a/docs/user/introduction/images/intro-spaces.png b/docs/user/introduction/images/intro-spaces.png index 6f3212cbde26e1..b2576cbc9acc4f 100644 Binary files a/docs/user/introduction/images/intro-spaces.png and b/docs/user/introduction/images/intro-spaces.png differ diff --git a/docs/user/management.asciidoc b/docs/user/management.asciidoc index c371aa695c4759..ee85819b4fd989 100644 --- a/docs/user/management.asciidoc +++ b/docs/user/management.asciidoc @@ -6,6 +6,10 @@ *Stack Management* is home to UIs for managing all things Elastic Stack— indices, clusters, licenses, UI settings, index patterns, spaces, and more. + +Access to individual features is governed by {es} and {kib} privileges. +Consult your administrator if you do not have the appropriate access. + [float] [[manage-ingest]] == Ingest diff --git a/docs/user/security/authorization/index.asciidoc b/docs/user/security/authorization/index.asciidoc index 150004b3ad6914..af5fd34b3b3509 100644 --- a/docs/user/security/authorization/index.asciidoc +++ b/docs/user/security/authorization/index.asciidoc @@ -12,7 +12,12 @@ NOTE: When running multiple tenants of {kib} by changing the `kibana.index` in y [[xpack-kibana-role-management]] === {kib} role management -To create a role that grants {kib} privileges, open the main menu, click *Stack Management > Roles*, then click *Create role*. +To create a role that grants {kib} privileges, open the menu, then click *Stack Management > Roles* and click **Create role**. + +[float] +==== Required permissions + +The `manage_security` cluster privilege is required to access role management. [[adding_kibana_privileges]] ==== Adding {kib} privileges diff --git a/docs/user/security/images/add-space-privileges.png b/docs/user/security/images/add-space-privileges.png old mode 100755 new mode 100644 index 7739332c33b603..d2fcbe76c1a061 Binary files a/docs/user/security/images/add-space-privileges.png and b/docs/user/security/images/add-space-privileges.png differ diff --git a/docs/user/security/images/assign_base_privilege.png b/docs/user/security/images/assign_base_privilege.png index 34e2bcf81d618c..93bed0de05555a 100644 Binary files a/docs/user/security/images/assign_base_privilege.png and b/docs/user/security/images/assign_base_privilege.png differ diff --git a/docs/user/security/images/assign_feature_privilege.png b/docs/user/security/images/assign_feature_privilege.png index c9449f6390253e..d42ec208325a33 100644 Binary files a/docs/user/security/images/assign_feature_privilege.png and b/docs/user/security/images/assign_feature_privilege.png differ diff --git a/docs/user/security/images/privilege-example-1.png b/docs/user/security/images/privilege-example-1.png old mode 100755 new mode 100644 index 68ba716437240e..b8fb4d15b8f77c Binary files a/docs/user/security/images/privilege-example-1.png and b/docs/user/security/images/privilege-example-1.png differ diff --git a/docs/user/security/images/role-space-visualization.png b/docs/user/security/images/role-space-visualization.png index 746af89c66e85d..0de94d81065fe5 100644 Binary files a/docs/user/security/images/role-space-visualization.png and b/docs/user/security/images/role-space-visualization.png differ diff --git a/docs/user/security/images/view-privilege-summary.png b/docs/user/security/images/view-privilege-summary.png old mode 100755 new mode 100644 index d93d55c93fd128..7d2f3018d7de9e Binary files a/docs/user/security/images/view-privilege-summary.png and b/docs/user/security/images/view-privilege-summary.png differ diff --git a/docs/user/security/index.asciidoc b/docs/user/security/index.asciidoc index b5ab57d8f525ac..18ace452ce00c1 100644 --- a/docs/user/security/index.asciidoc +++ b/docs/user/security/index.asciidoc @@ -10,6 +10,12 @@ auditing. For more information, see {ref}/secure-cluster.html[Secure a cluster] and <>. +[float] +=== Required permissions + +The `manage_security` cluster privilege is required to access all Security features. + + [float] === Users diff --git a/docs/user/security/role-mappings/index.asciidoc b/docs/user/security/role-mappings/index.asciidoc index 3f9a17e98d77fe..ca3ca9a686892a 100644 --- a/docs/user/security/role-mappings/index.asciidoc +++ b/docs/user/security/role-mappings/index.asciidoc @@ -19,6 +19,11 @@ With *Role mappings*, you can: [role="screenshot"] image:user/security/role-mappings/images/role-mappings-grid.png["Role mappings"] +[float] +==== Required permissions + +The `manage_security` cluster privilege is required to manage Role Mappings. + [float] === Create a role mapping diff --git a/packages/kbn-apm-config-loader/src/config.test.ts b/packages/kbn-apm-config-loader/src/config.test.ts index 83438215716acc..8adc4d84635af5 100644 --- a/packages/kbn-apm-config-loader/src/config.test.ts +++ b/packages/kbn-apm-config-loader/src/config.test.ts @@ -28,6 +28,8 @@ import { import { ApmConfiguration } from './config'; +const initialEnv = { ...process.env }; + describe('ApmConfiguration', () => { beforeEach(() => { packageMock.raw = { @@ -39,6 +41,7 @@ describe('ApmConfiguration', () => { }); afterEach(() => { + process.env = { ...initialEnv }; resetAllMocks(); }); @@ -90,13 +93,16 @@ describe('ApmConfiguration', () => { let config = new ApmConfiguration(mockedRootDir, {}, false); expect(config.getConfig('serviceName')).toEqual( expect.objectContaining({ - serverUrl: expect.any(String), - secretToken: expect.any(String), + breakdownMetrics: true, }) ); config = new ApmConfiguration(mockedRootDir, {}, true); - expect(Object.keys(config.getConfig('serviceName'))).not.toContain('serverUrl'); + expect(config.getConfig('serviceName')).toEqual( + expect.objectContaining({ + breakdownMetrics: false, + }) + ); }); it('loads the configuration from the kibana config file', () => { @@ -156,4 +162,32 @@ describe('ApmConfiguration', () => { }) ); }); + + it('correctly sets environment', () => { + delete process.env.ELASTIC_APM_ENVIRONMENT; + delete process.env.NODE_ENV; + + let config = new ApmConfiguration(mockedRootDir, {}, false); + expect(config.getConfig('serviceName')).toEqual( + expect.objectContaining({ + environment: 'development', + }) + ); + + process.env.NODE_ENV = 'production'; + config = new ApmConfiguration(mockedRootDir, {}, false); + expect(config.getConfig('serviceName')).toEqual( + expect.objectContaining({ + environment: 'production', + }) + ); + + process.env.ELASTIC_APM_ENVIRONMENT = 'ci'; + config = new ApmConfiguration(mockedRootDir, {}, false); + expect(config.getConfig('serviceName')).toEqual( + expect.objectContaining({ + environment: 'ci', + }) + ); + }); }); diff --git a/packages/kbn-apm-config-loader/src/config.ts b/packages/kbn-apm-config-loader/src/config.ts index 897e7fd7ca610c..a611e205ec83a9 100644 --- a/packages/kbn-apm-config-loader/src/config.ts +++ b/packages/kbn-apm-config-loader/src/config.ts @@ -26,32 +26,26 @@ import { readFileSync } from 'fs'; import { ApmAgentConfig } from './types'; const getDefaultConfig = (isDistributable: boolean): ApmAgentConfig => { - if (isDistributable) { - return { - active: false, - globalLabels: {}, - // Do not use a centralized controlled config - centralConfig: false, - // Capture all exceptions that are not caught - logUncaughtExceptions: true, - // Can be performance intensive, disabling by default - breakdownMetrics: false, - }; - } - + // https://www.elastic.co/guide/en/apm/agent/nodejs/current/configuration.html return { - active: false, - serverUrl: 'https://f1542b814f674090afd914960583265f.apm.us-central1.gcp.cloud.es.io:443', + active: process.env.ELASTIC_APM_ACTIVE || false, + environment: process.env.ELASTIC_APM_ENVIRONMENT || process.env.NODE_ENV || 'development', + + serverUrl: 'https://b1e3b4b4233e44cdad468c127d0af8d8.apm.europe-west1.gcp.cloud.es.io:443', + // The secretToken below is intended to be hardcoded in this file even though // it makes it public. This is not a security/privacy issue. Normally we'd // instead disable the need for a secretToken in the APM Server config where // the data is transmitted to, but due to how it's being hosted, it's easier, // for now, to simply leave it in. - secretToken: 'R0Gjg46pE9K9wGestd', + secretToken: '2OyjjaI6RVkzx2O5CV', + + logUncaughtExceptions: true, globalLabels: {}, - breakdownMetrics: true, centralConfig: false, - logUncaughtExceptions: true, + + // Can be performance intensive, disabling by default + breakdownMetrics: isDistributable ? false : true, }; }; @@ -84,7 +78,8 @@ export class ApmConfiguration { getDefaultConfig(this.isDistributable), this.getConfigFromKibanaConfig(), this.getDevConfig(), - this.getDistConfig() + this.getDistConfig(), + this.getCIConfig() ); const rev = this.getGitRev(); @@ -146,6 +141,21 @@ export class ApmConfiguration { }; } + private getCIConfig(): ApmAgentConfig { + if (process.env.ELASTIC_APM_ENVIRONMENT !== 'ci') { + return {}; + } + + return { + globalLabels: { + branch: process.env.ghprbSourceBranch || '', + targetBranch: process.env.ghprbTargetBranch || '', + ciJobName: process.env.JOB_NAME || '', + ciBuildNumber: process.env.BUILD_NUMBER || '', + }, + }; + } + private getGitRev() { if (this.isDistributable) { return this.pkgBuild.sha; diff --git a/packages/kbn-optimizer/limits.yml b/packages/kbn-optimizer/limits.yml index fd0be15affab3d..c660d372225042 100644 --- a/packages/kbn-optimizer/limits.yml +++ b/packages/kbn-optimizer/limits.yml @@ -14,7 +14,7 @@ pageLoadAssetSize: dashboard: 374194 dashboardEnhanced: 65646 dashboardMode: 22716 - data: 1170713 + data: 1287839 dataEnhanced: 50420 devTools: 38637 discover: 105145 diff --git a/packages/kbn-optimizer/src/cli.ts b/packages/kbn-optimizer/src/cli.ts index 28b3e37380b4e9..976ffe1f5b1b5f 100644 --- a/packages/kbn-optimizer/src/cli.ts +++ b/packages/kbn-optimizer/src/cli.ts @@ -146,7 +146,11 @@ run( await lastValueFrom(update$.pipe(logOptimizerState(log, config))); if (updateLimits) { - updateBundleLimits(log, config); + updateBundleLimits({ + log, + config, + dropMissing: !(focus || filter), + }); } }, { diff --git a/packages/kbn-optimizer/src/limits.ts b/packages/kbn-optimizer/src/limits.ts index b0fae0901251d4..64ec92657fc5b6 100644 --- a/packages/kbn-optimizer/src/limits.ts +++ b/packages/kbn-optimizer/src/limits.ts @@ -79,10 +79,18 @@ export function validateLimitsForAllBundles(log: ToolingLog, config: OptimizerCo log.success('limits.yml file valid'); } -export function updateBundleLimits(log: ToolingLog, config: OptimizerConfig) { +interface UpdateBundleLimitsOptions { + log: ToolingLog; + config: OptimizerConfig; + dropMissing: boolean; +} + +export function updateBundleLimits({ log, config, dropMissing }: UpdateBundleLimitsOptions) { const metrics = getMetrics(log, config); - const pageLoadAssetSize: NonNullable = {}; + const pageLoadAssetSize: NonNullable = dropMissing + ? {} + : config.limits.pageLoadAssetSize ?? {}; for (const metric of metrics.sort((a, b) => a.id.localeCompare(b.id))) { if (metric.group === 'page load bundle size') { diff --git a/packages/kbn-plugin-generator/README.md b/packages/kbn-plugin-generator/README.md index 9ff9a8aa95ca2d..bee8e6c2ca783b 100644 --- a/packages/kbn-plugin-generator/README.md +++ b/packages/kbn-plugin-generator/README.md @@ -51,7 +51,7 @@ yarn kbn bootstrap Generated plugins receive a handful of scripts that can be used during development. Those scripts are detailed in the [README.md](template/README.md) file in each newly generated plugin, and expose the scripts provided by the [Kibana plugin helpers](../kbn-plugin-helpers), but here is a quick reference in case you need it: -> ***NOTE:*** All of these scripts should be run from the generated plugin. +> ***NOTE:*** The following scripts should be run from the generated plugin. - `yarn kbn bootstrap` @@ -59,14 +59,6 @@ Generated plugins receive a handful of scripts that can be used during developme > ***IMPORTANT:*** Use this script instead of `yarn` to install dependencies when switching branches, and re-run it whenever your dependencies change. - - `yarn start` - - Start kibana and have it include this plugin. You can pass any arguments that you would normally send to `bin/kibana` - - ``` - yarn start --elasticsearch.hosts http://localhost:9220 - ``` - - `yarn build` Build a distributable archive of your plugin. @@ -75,4 +67,15 @@ Generated plugins receive a handful of scripts that can be used during developme Run the server tests using mocha. + +To start kibana run the following command from Kibana root. + + - `yarn start` + + Start kibana and it will automatically include this plugin. You can pass any arguments that you would normally send to `bin/kibana` + + ``` + yarn start --elasticsearch.hosts http://localhost:9220 + ``` + For more information about any of these commands run `yarn ${task} --help`. For a full list of tasks run `yarn run` or take a look in the `package.json` file. diff --git a/src/core/server/elasticsearch/client/client_config.test.ts b/src/core/server/elasticsearch/client/client_config.test.ts index e8083836d3c1ec..2beb07e6da5bc3 100644 --- a/src/core/server/elasticsearch/client/client_config.test.ts +++ b/src/core/server/elasticsearch/client/client_config.test.ts @@ -216,28 +216,14 @@ describe('parseClientOptions', () => { ); }); - it('adds auth to the nodes if both `username` and `password` are set', () => { - let options = parseClientOptions( + it('does not add auth to the nodes', () => { + const options = parseClientOptions( createConfig({ username: 'user', - hosts: ['http://node-A:9200'], - }), - false - ); - expect(options.nodes).toMatchInlineSnapshot(` - Array [ - Object { - "url": "http://node-a:9200/", - }, - ] - `); - - options = parseClientOptions( - createConfig({ password: 'pass', hosts: ['http://node-A:9200'], }), - false + true ); expect(options.nodes).toMatchInlineSnapshot(` Array [ @@ -246,22 +232,6 @@ describe('parseClientOptions', () => { }, ] `); - - options = parseClientOptions( - createConfig({ - username: 'user', - password: 'pass', - hosts: ['http://node-A:9200'], - }), - false - ); - expect(options.nodes).toMatchInlineSnapshot(` - Array [ - Object { - "url": "http://user:pass@node-a:9200/", - }, - ] - `); }); }); describe('when `scoped` is true', () => { diff --git a/src/core/server/elasticsearch/client/client_config.ts b/src/core/server/elasticsearch/client/client_config.ts index f24c0d86550b89..dab20edfc61ffe 100644 --- a/src/core/server/elasticsearch/client/client_config.ts +++ b/src/core/server/elasticsearch/client/client_config.ts @@ -93,7 +93,7 @@ export function parseClientOptions( }; } - clientOptions.nodes = config.hosts.map((host) => convertHost(host, !scoped, config)); + clientOptions.nodes = config.hosts.map((host) => convertHost(host)); if (config.ssl) { clientOptions.ssl = generateSslConfig( @@ -140,18 +140,10 @@ const generateSslConfig = ( return ssl; }; -const convertHost = ( - host: string, - needAuth: boolean, - { username, password }: ElasticsearchClientConfig -): NodeOptions => { +const convertHost = (host: string): NodeOptions => { const url = new URL(host); const isHTTPS = url.protocol === 'https:'; url.port = url.port || (isHTTPS ? '443' : '80'); - if (needAuth && username && password) { - url.username = username; - url.password = password; - } return { url, diff --git a/src/dev/ci_setup/setup_env.sh b/src/dev/ci_setup/setup_env.sh index 8ec80ac295c73c..5dac270239c4af 100644 --- a/src/dev/ci_setup/setup_env.sh +++ b/src/dev/ci_setup/setup_env.sh @@ -24,6 +24,10 @@ export NODE_OPTIONS="$NODE_OPTIONS --max-old-space-size=4096" ### export FORCE_COLOR=1 +### APM tracking +### +export ELASTIC_APM_ENVIRONMENT=ci + ### ### check that we seem to be in a kibana project ### diff --git a/src/fixtures/telemetry_collectors/externally_defined_collector.ts b/src/fixtures/telemetry_collectors/externally_defined_collector.ts index 00a8d643e27b33..decee9df28185e 100644 --- a/src/fixtures/telemetry_collectors/externally_defined_collector.ts +++ b/src/fixtures/telemetry_collectors/externally_defined_collector.ts @@ -16,7 +16,10 @@ * specific language governing permissions and limitations * under the License. */ -import { CollectorSet, CollectorOptions } from '../../plugins/usage_collection/server/collector'; +import { + CollectorSet, + UsageCollectorOptions, +} from '../../plugins/usage_collection/server/collector'; import { loggerMock } from '../../core/server/logging/logger.mock'; const collectorSet = new CollectorSet({ @@ -28,7 +31,7 @@ interface Usage { locale: string; } -function createCollector(): CollectorOptions { +function createCollector(): UsageCollectorOptions { return { type: 'from_fn_collector', isReady: () => true, @@ -46,7 +49,7 @@ function createCollector(): CollectorOptions { } export function defineCollectorFromVariable() { - const fromVarCollector: CollectorOptions = { + const fromVarCollector: UsageCollectorOptions = { type: 'from_variable_collector', isReady: () => true, fetch(): Usage { diff --git a/src/fixtures/telemetry_collectors/unmapped_collector.ts b/src/fixtures/telemetry_collectors/unmapped_collector.ts index 1ea360fcd9e960..143bb168aff351 100644 --- a/src/fixtures/telemetry_collectors/unmapped_collector.ts +++ b/src/fixtures/telemetry_collectors/unmapped_collector.ts @@ -28,6 +28,7 @@ interface Usage { locale: string; } +// @ts-expect-error Intentionally not specifying `schema` export const myCollector = makeUsageCollector({ type: 'unmapped_collector', isReady: () => true, diff --git a/src/legacy/server/i18n/constants.ts b/src/legacy/server/i18n/constants.ts index 96fa420d4c6e18..a7a410dbcb5b34 100644 --- a/src/legacy/server/i18n/constants.ts +++ b/src/legacy/server/i18n/constants.ts @@ -18,8 +18,3 @@ */ export const I18N_RC = '.i18nrc.json'; - -/** - * The type name used within the Monitoring index to publish localization stats. - */ -export const KIBANA_LOCALIZATION_STATS_TYPE = 'localization'; diff --git a/src/legacy/server/i18n/localization/telemetry_localization_collector.ts b/src/legacy/server/i18n/localization/telemetry_localization_collector.ts index 89566dfd4ef689..fb837f5ae28df9 100644 --- a/src/legacy/server/i18n/localization/telemetry_localization_collector.ts +++ b/src/legacy/server/i18n/localization/telemetry_localization_collector.ts @@ -21,7 +21,6 @@ import { i18nLoader } from '@kbn/i18n'; import { size } from 'lodash'; import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; import { getIntegrityHashes, Integrities } from './file_integrity'; -import { KIBANA_LOCALIZATION_STATS_TYPE } from '../constants'; export interface UsageStats { locale: string; @@ -63,14 +62,20 @@ export function createCollectorFetch({ }; } +// TODO: Migrate out of the Legacy dir export function registerLocalizationUsageCollector( usageCollection: UsageCollectionSetup, helpers: LocalizationUsageCollectorHelpers ) { - const collector = usageCollection.makeUsageCollector({ - type: KIBANA_LOCALIZATION_STATS_TYPE, + const collector = usageCollection.makeUsageCollector({ + type: 'localization', isReady: () => true, fetch: createCollectorFetch(helpers), + schema: { + locale: { type: 'keyword' }, + integrities: { DYNAMIC_KEY: { type: 'text' } }, + labelsCount: { type: 'long' }, + }, }); usageCollection.registerCollector(collector); diff --git a/src/plugins/advanced_settings/public/management_app/components/field/field.tsx b/src/plugins/advanced_settings/public/management_app/components/field/field.tsx index 794168132abb27..e9fa2833c3db5f 100644 --- a/src/plugins/advanced_settings/public/management_app/components/field/field.tsx +++ b/src/plugins/advanced_settings/public/management_app/components/field/field.tsx @@ -22,6 +22,7 @@ import classNames from 'classnames'; import 'brace/theme/textmate'; import 'brace/mode/markdown'; +import 'brace/mode/json'; import { EuiBadge, diff --git a/src/plugins/data/public/public.api.md b/src/plugins/data/public/public.api.md index d2439e3f1573c9..81fa6d4ba20db7 100644 --- a/src/plugins/data/public/public.api.md +++ b/src/plugins/data/public/public.api.md @@ -1598,13 +1598,13 @@ export interface OptionedValueProp { value: string; } -// Warning: (ae-forgotten-export) The symbol "KbnError" needs to be exported by the entry point index.d.ts +// Warning: (ae-forgotten-export) The symbol "EsError" needs to be exported by the entry point index.d.ts // Warning: (ae-missing-release-tag) "PainlessError" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) -export class PainlessError extends KbnError { - // Warning: (ae-forgotten-export) The symbol "EsError" needs to be exported by the entry point index.d.ts - constructor(err: EsError, request: IKibanaSearchRequest); +export class PainlessError extends EsError { + // Warning: (ae-forgotten-export) The symbol "IEsError" needs to be exported by the entry point index.d.ts + constructor(err: IEsError, request: IKibanaSearchRequest); // (undocumented) getErrorMessage(application: ApplicationStart): JSX.Element; // (undocumented) @@ -2134,6 +2134,7 @@ export interface SearchSourceFields { version?: boolean; } +// Warning: (ae-forgotten-export) The symbol "KbnError" needs to be exported by the entry point index.d.ts // Warning: (ae-missing-release-tag) "SearchTimeoutError" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public diff --git a/src/plugins/data/public/search/errors/es_error.tsx b/src/plugins/data/public/search/errors/es_error.tsx new file mode 100644 index 00000000000000..53d00159b836b3 --- /dev/null +++ b/src/plugins/data/public/search/errors/es_error.tsx @@ -0,0 +1,46 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import React from 'react'; +import { EuiCodeBlock, EuiSpacer } from '@elastic/eui'; +import { ApplicationStart } from 'kibana/public'; +import { KbnError } from '../../../../kibana_utils/common'; +import { IEsError } from './types'; +import { getRootCause } from './utils'; + +export class EsError extends KbnError { + constructor(protected readonly err: IEsError) { + super('EsError'); + } + + public getErrorMessage(application: ApplicationStart) { + const rootCause = getRootCause(this.err)?.reason; + + return ( + <> + + {rootCause ? ( + + {rootCause} + + ) : null} + + ); + } +} diff --git a/src/plugins/data/public/search/errors/http_error.tsx b/src/plugins/data/public/search/errors/http_error.tsx new file mode 100644 index 00000000000000..58ae3148804a27 --- /dev/null +++ b/src/plugins/data/public/search/errors/http_error.tsx @@ -0,0 +1,38 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { EuiCodeBlock, EuiSpacer } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import React from 'react'; + +export function getHttpError(message: string) { + return ( + <> + {i18n.translate('data.errors.fetchError', { + defaultMessage: + 'Check your network and proxy configuration. If the problem persists, contact your network administrator.', + })} + + + + {message} + + + ); +} diff --git a/src/plugins/data/public/search/errors/index.ts b/src/plugins/data/public/search/errors/index.ts index 6082e758a8badb..01357d25334a3f 100644 --- a/src/plugins/data/public/search/errors/index.ts +++ b/src/plugins/data/public/search/errors/index.ts @@ -17,5 +17,9 @@ * under the License. */ +export * from './es_error'; export * from './painless_error'; export * from './timeout_error'; +export * from './utils'; +export * from './types'; +export * from './http_error'; diff --git a/src/plugins/data/public/search/errors/painless_error.tsx b/src/plugins/data/public/search/errors/painless_error.tsx index 244f205469a2f4..282a602d358c77 100644 --- a/src/plugins/data/public/search/errors/painless_error.tsx +++ b/src/plugins/data/public/search/errors/painless_error.tsx @@ -22,22 +22,15 @@ import { i18n } from '@kbn/i18n'; import { EuiButton, EuiSpacer, EuiText, EuiCodeBlock } from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n/react'; import { ApplicationStart } from 'kibana/public'; -import { KbnError } from '../../../../kibana_utils/common'; -import { EsError, isEsError } from './types'; +import { IEsError, isEsError } from './types'; +import { EsError } from './es_error'; +import { getRootCause } from './utils'; import { IKibanaSearchRequest } from '..'; -export class PainlessError extends KbnError { +export class PainlessError extends EsError { painlessStack?: string; - constructor(err: EsError, request: IKibanaSearchRequest) { - const rootCause = getRootCause(err as EsError); - - super( - i18n.translate('data.painlessError.painlessScriptedFieldErrorMessage', { - defaultMessage: "Error executing Painless script: '{script}'.", - values: { script: rootCause?.script }, - }) - ); - this.painlessStack = rootCause?.script_stack ? rootCause?.script_stack.join('\n') : undefined; + constructor(err: IEsError, request: IKibanaSearchRequest) { + super(err); } public getErrorMessage(application: ApplicationStart) { @@ -47,14 +40,20 @@ export class PainlessError extends KbnError { }); } + const rootCause = getRootCause(this.err); + const painlessStack = rootCause?.script_stack ? rootCause?.script_stack.join('\n') : undefined; + return ( <> - {this.message} + {i18n.translate('data.painlessError.painlessScriptedFieldErrorMessage', { + defaultMessage: "Error executing Painless script: '{script}'.", + values: { script: rootCause?.script }, + })} - {this.painlessStack ? ( + {painlessStack ? ( - {this.painlessStack} + {painlessStack} ) : null} @@ -67,21 +66,10 @@ export class PainlessError extends KbnError { } } -function getFailedShards(err: EsError) { - const failedShards = - err.body?.attributes?.error?.failed_shards || - err.body?.attributes?.error?.caused_by?.failed_shards; - return failedShards ? failedShards[0] : undefined; -} - -function getRootCause(err: EsError) { - return getFailedShards(err)?.reason; -} - -export function isPainlessError(err: Error | EsError) { +export function isPainlessError(err: Error | IEsError) { if (!isEsError(err)) return false; - const rootCause = getRootCause(err as EsError); + const rootCause = getRootCause(err as IEsError); if (!rootCause) return false; const { lang } = rootCause; diff --git a/src/plugins/data/public/search/errors/types.ts b/src/plugins/data/public/search/errors/types.ts index 4182209eb68a5a..011af015318a7a 100644 --- a/src/plugins/data/public/search/errors/types.ts +++ b/src/plugins/data/public/search/errors/types.ts @@ -17,7 +17,7 @@ * under the License. */ -interface FailedShard { +export interface FailedShard { shard: number; index: string; node: string; @@ -39,7 +39,7 @@ interface FailedShard { }; } -export interface EsError { +export interface IEsError { body: { statusCode: number; error: string; @@ -68,6 +68,6 @@ export interface EsError { }; } -export function isEsError(e: any): e is EsError { +export function isEsError(e: any): e is IEsError { return !!e.body?.attributes; } diff --git a/src/plugins/data/public/search/errors/utils.ts b/src/plugins/data/public/search/errors/utils.ts new file mode 100644 index 00000000000000..d07d9b05e91e9d --- /dev/null +++ b/src/plugins/data/public/search/errors/utils.ts @@ -0,0 +1,31 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { IEsError } from './types'; + +export function getFailedShards(err: IEsError) { + const failedShards = + err.body?.attributes?.error?.failed_shards || + err.body?.attributes?.error?.caused_by?.failed_shards; + return failedShards ? failedShards[0] : undefined; +} + +export function getRootCause(err: IEsError) { + return getFailedShards(err)?.reason; +} diff --git a/src/plugins/data/public/search/search_interceptor.ts b/src/plugins/data/public/search/search_interceptor.ts index e3c6dd3e287d44..1afcf4615ab5a1 100644 --- a/src/plugins/data/public/search/search_interceptor.ts +++ b/src/plugins/data/public/search/search_interceptor.ts @@ -21,6 +21,7 @@ import { get, memoize, trimEnd } from 'lodash'; import { BehaviorSubject, throwError, timer, defer, from, Observable, NEVER } from 'rxjs'; import { catchError, finalize } from 'rxjs/operators'; import { CoreStart, CoreSetup, ToastsSetup } from 'kibana/public'; +import { i18n } from '@kbn/i18n'; import { getCombinedSignal, AbortError, @@ -31,7 +32,15 @@ import { ISessionService, } from '../../common'; import { SearchUsageCollector } from './collectors'; -import { SearchTimeoutError, PainlessError, isPainlessError, TimeoutErrorMode } from './errors'; +import { + SearchTimeoutError, + PainlessError, + isPainlessError, + TimeoutErrorMode, + isEsError, + EsError, + getHttpError, +} from './errors'; import { toMountPoint } from '../../../kibana_react/public'; export interface SearchInterceptorDeps { @@ -101,8 +110,12 @@ export class SearchInterceptor { } else if (options?.abortSignal?.aborted) { // In the case an application initiated abort, throw the existing AbortError. return e; - } else if (isPainlessError(e)) { - return new PainlessError(e, request); + } else if (isEsError(e)) { + if (isPainlessError(e)) { + return new PainlessError(e, request); + } else { + return new EsError(e); + } } else { return e; } @@ -236,24 +249,28 @@ export class SearchInterceptor { * */ public showError(e: Error) { - if (e instanceof AbortError) return; - - if (e instanceof SearchTimeoutError) { + if (e instanceof AbortError || e instanceof SearchTimeoutError) { // The SearchTimeoutError is shown by the interceptor in getSearchError (regardless of how the app chooses to handle errors) return; - } - - if (e instanceof PainlessError) { + } else if (e instanceof EsError) { this.deps.toasts.addDanger({ - title: 'Search Error', + title: i18n.translate('data.search.esErrorTitle', { + defaultMessage: 'Cannot retrieve search results', + }), text: toMountPoint(e.getErrorMessage(this.application)), }); - return; + } else if (e.constructor.name === 'HttpFetchError') { + this.deps.toasts.addDanger({ + title: i18n.translate('data.search.httpErrorTitle', { + defaultMessage: 'Cannot retrieve your data', + }), + text: toMountPoint(getHttpError(e.message)), + }); + } else { + this.deps.toasts.addError(e, { + title: 'Search Error', + }); } - - this.deps.toasts.addError(e, { - title: 'Search Error', - }); } } diff --git a/src/plugins/data/public/ui/query_string_input/query_string_input.tsx b/src/plugins/data/public/ui/query_string_input/query_string_input.tsx index c17872028ea8db..bc9e2ed6a83ceb 100644 --- a/src/plugins/data/public/ui/query_string_input/query_string_input.tsx +++ b/src/plugins/data/public/ui/query_string_input/query_string_input.tsx @@ -111,6 +111,7 @@ export default class QueryStringInputUI extends Component { private persistedLog: PersistedLog | undefined; private abortController?: AbortController; + private fetchIndexPatternsAbortController?: AbortController; private services = this.props.kibana.services; private componentIsUnmounting = false; private queryBarInputDivRefInstance: RefObject = createRef(); @@ -119,7 +120,7 @@ export default class QueryStringInputUI extends Component { return toUser(this.props.query.query); }; - private fetchIndexPatterns = async () => { + private fetchIndexPatterns = debounce(async () => { const stringPatterns = this.props.indexPatterns.filter( (indexPattern) => typeof indexPattern === 'string' ) as string[]; @@ -127,16 +128,26 @@ export default class QueryStringInputUI extends Component { (indexPattern) => typeof indexPattern !== 'string' ) as IIndexPattern[]; + // abort the previous fetch to avoid overriding with outdated data + // issue https://github.com/elastic/kibana/issues/80831 + if (this.fetchIndexPatternsAbortController) this.fetchIndexPatternsAbortController.abort(); + this.fetchIndexPatternsAbortController = new AbortController(); + const currentAbortController = this.fetchIndexPatternsAbortController; + const objectPatternsFromStrings = (await fetchIndexPatterns( this.services.savedObjects!.client, stringPatterns, this.services.uiSettings! )) as IIndexPattern[]; - this.setState({ - indexPatterns: [...objectPatterns, ...objectPatternsFromStrings], - }); - }; + if (!currentAbortController.signal.aborted) { + this.setState({ + indexPatterns: [...objectPatterns, ...objectPatternsFromStrings], + }); + + this.updateSuggestions(); + } + }, 200); private getSuggestions = async () => { if (!this.inputRef) { @@ -506,7 +517,7 @@ export default class QueryStringInputUI extends Component { } this.initPersistedLog(); - this.fetchIndexPatterns().then(this.updateSuggestions); + this.fetchIndexPatterns(); this.handleListUpdate(); window.addEventListener('resize', this.handleAutoHeight); @@ -525,7 +536,7 @@ export default class QueryStringInputUI extends Component { this.initPersistedLog(); if (!isEqual(prevProps.indexPatterns, this.props.indexPatterns)) { - this.fetchIndexPatterns().then(this.updateSuggestions); + this.fetchIndexPatterns(); } else if (!isEqual(prevProps.query, this.props.query)) { this.updateSuggestions(); } diff --git a/src/plugins/data/server/index_patterns/fetcher/index_patterns_fetcher.ts b/src/plugins/data/server/index_patterns/fetcher/index_patterns_fetcher.ts index 57c636a9e3c69c..e75b8761984ec9 100644 --- a/src/plugins/data/server/index_patterns/fetcher/index_patterns_fetcher.ts +++ b/src/plugins/data/server/index_patterns/fetcher/index_patterns_fetcher.ts @@ -17,7 +17,7 @@ * under the License. */ -import { LegacyAPICaller } from 'kibana/server'; +import { ElasticsearchClient } from 'kibana/server'; import { getFieldCapabilities, resolveTimePattern, createNoMatchingIndicesError } from './lib'; @@ -37,10 +37,12 @@ interface FieldSubType { } export class IndexPatternsFetcher { - private _callDataCluster: LegacyAPICaller; + private elasticsearchClient: ElasticsearchClient; + private allowNoIndices: boolean; - constructor(callDataCluster: LegacyAPICaller) { - this._callDataCluster = callDataCluster; + constructor(elasticsearchClient: ElasticsearchClient, allowNoIndices: boolean = false) { + this.elasticsearchClient = elasticsearchClient; + this.allowNoIndices = allowNoIndices; } /** @@ -55,10 +57,12 @@ export class IndexPatternsFetcher { async getFieldsForWildcard(options: { pattern: string | string[]; metaFields?: string[]; - fieldCapsOptions?: { allowNoIndices: boolean }; + fieldCapsOptions?: { allow_no_indices: boolean }; }): Promise { const { pattern, metaFields, fieldCapsOptions } = options; - return await getFieldCapabilities(this._callDataCluster, pattern, metaFields, fieldCapsOptions); + return await getFieldCapabilities(this.elasticsearchClient, pattern, metaFields, { + allow_no_indices: fieldCapsOptions ? fieldCapsOptions.allow_no_indices : this.allowNoIndices, + }); } /** @@ -78,11 +82,11 @@ export class IndexPatternsFetcher { interval: string; }) { const { pattern, lookBack, metaFields } = options; - const { matches } = await resolveTimePattern(this._callDataCluster, pattern); + const { matches } = await resolveTimePattern(this.elasticsearchClient, pattern); const indices = matches.slice(0, lookBack); if (indices.length === 0) { throw createNoMatchingIndicesError(pattern); } - return await getFieldCapabilities(this._callDataCluster, indices, metaFields); + return await getFieldCapabilities(this.elasticsearchClient, indices, metaFields); } } diff --git a/src/plugins/data/server/index_patterns/fetcher/lib/es_api.test.js b/src/plugins/data/server/index_patterns/fetcher/lib/es_api.test.js index 8078ea32187b37..fad20a8f0be06c 100644 --- a/src/plugins/data/server/index_patterns/fetcher/lib/es_api.test.js +++ b/src/plugins/data/server/index_patterns/fetcher/lib/es_api.test.js @@ -32,36 +32,60 @@ describe('server/index_patterns/service/lib/es_api', () => { afterEach(() => sandbox.restore()); it('calls indices.getAlias() via callCluster', async () => { - const callCluster = sinon.stub(); + const getAlias = sinon.stub(); + const callCluster = { + indices: { + getAlias, + }, + fieldCaps: sinon.stub(), + }; + await callIndexAliasApi(callCluster); - sinon.assert.calledOnce(callCluster); - sinon.assert.calledWith(callCluster, 'indices.getAlias'); + sinon.assert.calledOnce(getAlias); }); it('passes indices directly to es api', async () => { const football = {}; - const callCluster = sinon.stub(); + const getAlias = sinon.stub(); + const callCluster = { + indices: { + getAlias, + }, + fieldCaps: sinon.stub(), + }; await callIndexAliasApi(callCluster, football); - sinon.assert.calledOnce(callCluster); - expect(callCluster.args[0][1].index).toBe(football); + sinon.assert.calledOnce(getAlias); + expect(getAlias.args[0][0].index).toBe(football); }); it('returns the es response directly', async () => { const football = {}; - const callCluster = sinon.stub().returns(football); + const getAlias = sinon.stub().returns(football); + const callCluster = { + indices: { + getAlias, + }, + fieldCaps: sinon.stub(), + }; const resp = await callIndexAliasApi(callCluster); - sinon.assert.calledOnce(callCluster); + sinon.assert.calledOnce(getAlias); expect(resp).toBe(football); }); it('sets ignoreUnavailable and allowNoIndices params', async () => { - const callCluster = sinon.stub(); + const getAlias = sinon.stub(); + const callCluster = { + indices: { + getAlias, + }, + fieldCaps: sinon.stub(), + }; await callIndexAliasApi(callCluster); - sinon.assert.calledOnce(callCluster); + sinon.assert.calledOnce(getAlias); - const passedOpts = callCluster.args[0][1]; - expect(passedOpts).toHaveProperty('ignoreUnavailable', true); - expect(passedOpts).toHaveProperty('allowNoIndices', false); + const passedOpts = getAlias.args[0][0]; + expect(passedOpts).toHaveProperty('ignore_unavailable', true); + expect(passedOpts).toHaveProperty('allow_no_indices', false); }); it('handles errors with convertEsError()', async () => { @@ -70,9 +94,15 @@ describe('server/index_patterns/service/lib/es_api', () => { const convertedError = new Error('convertedError'); sandbox.stub(convertEsErrorNS, 'convertEsError').throws(convertedError); - const callCluster = sinon.spy(async () => { + const getAlias = sinon.stub(async () => { throw esError; }); + const callCluster = { + indices: { + getAlias, + }, + fieldCaps: sinon.stub(), + }; try { await callIndexAliasApi(callCluster, indices); throw new Error('expected callIndexAliasApi() to throw'); @@ -91,37 +121,60 @@ describe('server/index_patterns/service/lib/es_api', () => { afterEach(() => sandbox.restore()); it('calls fieldCaps() via callCluster', async () => { - const callCluster = sinon.stub(); + const fieldCaps = sinon.stub(); + const callCluster = { + indices: { + getAlias: sinon.stub(), + }, + fieldCaps, + }; await callFieldCapsApi(callCluster); - sinon.assert.calledOnce(callCluster); - sinon.assert.calledWith(callCluster, 'fieldCaps'); + sinon.assert.calledOnce(fieldCaps); }); it('passes indices directly to es api', async () => { const football = {}; - const callCluster = sinon.stub(); + const fieldCaps = sinon.stub(); + const callCluster = { + indices: { + getAlias: sinon.stub(), + }, + fieldCaps, + }; await callFieldCapsApi(callCluster, football); - sinon.assert.calledOnce(callCluster); - expect(callCluster.args[0][1].index).toBe(football); + sinon.assert.calledOnce(fieldCaps); + expect(fieldCaps.args[0][0].index).toBe(football); }); it('returns the es response directly', async () => { const football = {}; - const callCluster = sinon.stub().returns(football); + const fieldCaps = sinon.stub().returns(football); + const callCluster = { + indices: { + getAlias: sinon.stub(), + }, + fieldCaps, + }; const resp = await callFieldCapsApi(callCluster); - sinon.assert.calledOnce(callCluster); + sinon.assert.calledOnce(fieldCaps); expect(resp).toBe(football); }); it('sets ignoreUnavailable, allowNoIndices, and fields params', async () => { - const callCluster = sinon.stub(); + const fieldCaps = sinon.stub(); + const callCluster = { + indices: { + getAlias: sinon.stub(), + }, + fieldCaps, + }; await callFieldCapsApi(callCluster); - sinon.assert.calledOnce(callCluster); + sinon.assert.calledOnce(fieldCaps); - const passedOpts = callCluster.args[0][1]; + const passedOpts = fieldCaps.args[0][0]; expect(passedOpts).toHaveProperty('fields', '*'); - expect(passedOpts).toHaveProperty('ignoreUnavailable', true); - expect(passedOpts).toHaveProperty('allowNoIndices', false); + expect(passedOpts).toHaveProperty('ignore_unavailable', true); + expect(passedOpts).toHaveProperty('allow_no_indices', false); }); it('handles errors with convertEsError()', async () => { @@ -130,9 +183,15 @@ describe('server/index_patterns/service/lib/es_api', () => { const convertedError = new Error('convertedError'); sandbox.stub(convertEsErrorNS, 'convertEsError').throws(convertedError); - const callCluster = sinon.spy(async () => { + const fieldCaps = sinon.spy(async () => { throw esError; }); + const callCluster = { + indices: { + getAlias: sinon.stub(), + }, + fieldCaps, + }; try { await callFieldCapsApi(callCluster, indices); throw new Error('expected callFieldCapsApi() to throw'); diff --git a/src/plugins/data/server/index_patterns/fetcher/lib/es_api.ts b/src/plugins/data/server/index_patterns/fetcher/lib/es_api.ts index 27ce14f9a3597c..7969324943a9fc 100644 --- a/src/plugins/data/server/index_patterns/fetcher/lib/es_api.ts +++ b/src/plugins/data/server/index_patterns/fetcher/lib/es_api.ts @@ -17,7 +17,7 @@ * under the License. */ -import { LegacyAPICaller } from 'kibana/server'; +import { ElasticsearchClient } from 'kibana/server'; import { convertEsError } from './errors'; import { FieldCapsResponse } from './field_capabilities'; @@ -46,15 +46,15 @@ export interface IndexAliasResponse { * @return {Promise} */ export async function callIndexAliasApi( - callCluster: LegacyAPICaller, + callCluster: ElasticsearchClient, indices: string[] | string -): Promise { +) { try { - return (await callCluster('indices.getAlias', { + return await callCluster.indices.getAlias({ index: indices, - ignoreUnavailable: true, - allowNoIndices: false, - })) as Promise; + ignore_unavailable: true, + allow_no_indices: false, + }); } catch (error) { throw convertEsError(indices, error); } @@ -73,17 +73,17 @@ export async function callIndexAliasApi( * @return {Promise} */ export async function callFieldCapsApi( - callCluster: LegacyAPICaller, + callCluster: ElasticsearchClient, indices: string[] | string, - fieldCapsOptions: { allowNoIndices: boolean } = { allowNoIndices: false } + fieldCapsOptions: { allow_no_indices: boolean } = { allow_no_indices: false } ) { try { - return (await callCluster('fieldCaps', { + return await callCluster.fieldCaps({ index: indices, fields: '*', - ignoreUnavailable: true, + ignore_unavailable: true, ...fieldCapsOptions, - })) as FieldCapsResponse; + }); } catch (error) { throw convertEsError(indices, error); } diff --git a/src/plugins/data/server/index_patterns/fetcher/lib/field_capabilities/field_capabilities.test.js b/src/plugins/data/server/index_patterns/fetcher/lib/field_capabilities/field_capabilities.test.js index 0e5757b7b782b5..2d860dc8b18433 100644 --- a/src/plugins/data/server/index_patterns/fetcher/lib/field_capabilities/field_capabilities.test.js +++ b/src/plugins/data/server/index_patterns/fetcher/lib/field_capabilities/field_capabilities.test.js @@ -48,9 +48,11 @@ describe('index_patterns/field_capabilities/field_capabilities', () => { }; const stubDeps = (options = {}) => { - const { esResponse = {}, fieldsFromFieldCaps = [], mergeOverrides = identity } = options; + const { esResponse = [], fieldsFromFieldCaps = [], mergeOverrides = identity } = options; - sandbox.stub(callFieldCapsApiNS, 'callFieldCapsApi').callsFake(async () => esResponse); + sandbox + .stub(callFieldCapsApiNS, 'callFieldCapsApi') + .callsFake(async () => ({ body: esResponse })); sandbox.stub(readFieldCapsResponseNS, 'readFieldCapsResponse').returns(fieldsFromFieldCaps); sandbox.stub(mergeOverridesNS, 'mergeOverrides').callsFake(mergeOverrides); }; diff --git a/src/plugins/data/server/index_patterns/fetcher/lib/field_capabilities/field_capabilities.ts b/src/plugins/data/server/index_patterns/fetcher/lib/field_capabilities/field_capabilities.ts index 62e77e0adad66a..b9e3e8aae0899e 100644 --- a/src/plugins/data/server/index_patterns/fetcher/lib/field_capabilities/field_capabilities.ts +++ b/src/plugins/data/server/index_patterns/fetcher/lib/field_capabilities/field_capabilities.ts @@ -19,9 +19,9 @@ import { defaults, keyBy, sortBy } from 'lodash'; -import { LegacyAPICaller } from 'kibana/server'; +import { ElasticsearchClient } from 'kibana/server'; import { callFieldCapsApi } from '../es_api'; -import { FieldCapsResponse, readFieldCapsResponse } from './field_caps_response'; +import { readFieldCapsResponse } from './field_caps_response'; import { mergeOverrides } from './overrides'; import { FieldDescriptor } from '../../index_patterns_fetcher'; @@ -36,17 +36,13 @@ import { FieldDescriptor } from '../../index_patterns_fetcher'; * @return {Promise>} */ export async function getFieldCapabilities( - callCluster: LegacyAPICaller, + callCluster: ElasticsearchClient, indices: string | string[] = [], metaFields: string[] = [], - fieldCapsOptions?: { allowNoIndices: boolean } + fieldCapsOptions?: { allow_no_indices: boolean } ) { - const esFieldCaps: FieldCapsResponse = await callFieldCapsApi( - callCluster, - indices, - fieldCapsOptions - ); - const fieldsFromFieldCapsByName = keyBy(readFieldCapsResponse(esFieldCaps), 'name'); + const esFieldCaps = await callFieldCapsApi(callCluster, indices, fieldCapsOptions); + const fieldsFromFieldCapsByName = keyBy(readFieldCapsResponse(esFieldCaps.body), 'name'); const allFieldsUnsorted = Object.keys(fieldsFromFieldCapsByName) .filter((name) => !name.startsWith('_')) diff --git a/src/plugins/data/server/index_patterns/fetcher/lib/resolve_time_pattern.test.js b/src/plugins/data/server/index_patterns/fetcher/lib/resolve_time_pattern.test.js index 660e9ec30db6a3..87f222aaad89dc 100644 --- a/src/plugins/data/server/index_patterns/fetcher/lib/resolve_time_pattern.test.js +++ b/src/plugins/data/server/index_patterns/fetcher/lib/resolve_time_pattern.test.js @@ -32,6 +32,11 @@ const TIME_PATTERN = '[logs-]dddd-YYYY.w'; describe('server/index_patterns/service/lib/resolve_time_pattern', () => { let sandbox; + const esClientMock = { + indices: { + getAlias: () => ({}), + }, + }; beforeEach(() => (sandbox = sinon.createSandbox())); afterEach(() => sandbox.restore()); @@ -39,7 +44,7 @@ describe('server/index_patterns/service/lib/resolve_time_pattern', () => { describe('pre request', () => { it('uses callIndexAliasApi() fn', async () => { sandbox.stub(callIndexAliasApiNS, 'callIndexAliasApi').returns({}); - await resolveTimePattern(noop, TIME_PATTERN); + await resolveTimePattern(esClientMock, TIME_PATTERN); sinon.assert.calledOnce(callIndexAliasApi); }); @@ -49,7 +54,7 @@ describe('server/index_patterns/service/lib/resolve_time_pattern', () => { sandbox.stub(timePatternToWildcardNS, 'timePatternToWildcard').returns(wildcard); - await resolveTimePattern(noop, timePattern); + await resolveTimePattern(esClientMock, timePattern); sinon.assert.calledOnce(timePatternToWildcard); expect(timePatternToWildcard.firstCall.args).toEqual([timePattern]); }); @@ -61,7 +66,7 @@ describe('server/index_patterns/service/lib/resolve_time_pattern', () => { sandbox.stub(callIndexAliasApiNS, 'callIndexAliasApi').returns({}); sandbox.stub(timePatternToWildcardNS, 'timePatternToWildcard').returns(wildcard); - await resolveTimePattern(noop, timePattern); + await resolveTimePattern(esClientMock, timePattern); sinon.assert.calledOnce(callIndexAliasApi); expect(callIndexAliasApi.firstCall.args[1]).toBe(wildcard); }); @@ -70,13 +75,15 @@ describe('server/index_patterns/service/lib/resolve_time_pattern', () => { describe('read response', () => { it('returns all aliases names in result.all, ordered by time desc', async () => { sandbox.stub(callIndexAliasApiNS, 'callIndexAliasApi').returns({ - 'logs-2016.2': {}, - 'logs-Saturday-2017.1': {}, - 'logs-2016.1': {}, - 'logs-Sunday-2017.1': {}, - 'logs-2015': {}, - 'logs-2016.3': {}, - 'logs-Friday-2017.1': {}, + body: { + 'logs-2016.2': {}, + 'logs-Saturday-2017.1': {}, + 'logs-2016.1': {}, + 'logs-Sunday-2017.1': {}, + 'logs-2015': {}, + 'logs-2016.3': {}, + 'logs-Friday-2017.1': {}, + }, }); const resp = await resolveTimePattern(noop, TIME_PATTERN); @@ -94,13 +101,15 @@ describe('server/index_patterns/service/lib/resolve_time_pattern', () => { it('returns all indices matching the time pattern in matches, ordered by time desc', async () => { sandbox.stub(callIndexAliasApiNS, 'callIndexAliasApi').returns({ - 'logs-2016.2': {}, - 'logs-Saturday-2017.1': {}, - 'logs-2016.1': {}, - 'logs-Sunday-2017.1': {}, - 'logs-2015': {}, - 'logs-2016.3': {}, - 'logs-Friday-2017.1': {}, + body: { + 'logs-2016.2': {}, + 'logs-Saturday-2017.1': {}, + 'logs-2016.1': {}, + 'logs-Sunday-2017.1': {}, + 'logs-2015': {}, + 'logs-2016.3': {}, + 'logs-Friday-2017.1': {}, + }, }); const resp = await resolveTimePattern(noop, TIME_PATTERN); diff --git a/src/plugins/data/server/index_patterns/fetcher/lib/resolve_time_pattern.ts b/src/plugins/data/server/index_patterns/fetcher/lib/resolve_time_pattern.ts index 2e408d7569be56..95ec06dd9c6e66 100644 --- a/src/plugins/data/server/index_patterns/fetcher/lib/resolve_time_pattern.ts +++ b/src/plugins/data/server/index_patterns/fetcher/lib/resolve_time_pattern.ts @@ -20,7 +20,7 @@ import { chain } from 'lodash'; import moment from 'moment'; -import { LegacyAPICaller } from 'kibana/server'; +import { ElasticsearchClient } from 'kibana/server'; import { timePatternToWildcard } from './time_pattern_to_wildcard'; import { callIndexAliasApi, IndicesAliasResponse } from './es_api'; @@ -36,10 +36,10 @@ import { callIndexAliasApi, IndicesAliasResponse } from './es_api'; * and the indices that actually match the time * pattern (matches); */ -export async function resolveTimePattern(callCluster: LegacyAPICaller, timePattern: string) { +export async function resolveTimePattern(callCluster: ElasticsearchClient, timePattern: string) { const aliases = await callIndexAliasApi(callCluster, timePatternToWildcard(timePattern)); - const allIndexDetails = chain(aliases) + const allIndexDetails = chain(aliases.body) .reduce( (acc: string[], index: any, indexName: string) => acc.concat(indexName, Object.keys(index.aliases || {})), diff --git a/src/plugins/data/server/index_patterns/routes.ts b/src/plugins/data/server/index_patterns/routes.ts index 428e7fef6deead..041eb235d01e08 100644 --- a/src/plugins/data/server/index_patterns/routes.ts +++ b/src/plugins/data/server/index_patterns/routes.ts @@ -46,8 +46,8 @@ export function registerRoutes(http: HttpServiceSetup) { }, }, async (context, request, response) => { - const { callAsCurrentUser } = context.core.elasticsearch.legacy.client; - const indexPatterns = new IndexPatternsFetcher(callAsCurrentUser); + const { asCurrentUser } = context.core.elasticsearch.client; + const indexPatterns = new IndexPatternsFetcher(asCurrentUser); const { pattern, meta_fields: metaFields } = request.query; let parsedFields: string[] = []; @@ -105,8 +105,8 @@ export function registerRoutes(http: HttpServiceSetup) { }, }, async (context: RequestHandlerContext, request: any, response: any) => { - const { callAsCurrentUser } = context.core.elasticsearch.legacy.client; - const indexPatterns = new IndexPatternsFetcher(callAsCurrentUser); + const { asCurrentUser } = context.core.elasticsearch.client; + const indexPatterns = new IndexPatternsFetcher(asCurrentUser); const { pattern, interval, look_back: lookBack, meta_fields: metaFields } = request.query; let parsedFields: string[] = []; diff --git a/src/plugins/data/server/server.api.md b/src/plugins/data/server/server.api.md index 3143d5baa5b774..97cbb40c13db09 100644 --- a/src/plugins/data/server/server.api.md +++ b/src/plugins/data/server/server.api.md @@ -26,6 +26,7 @@ import { ISearchSource } from 'src/plugins/data/public'; import { KibanaRequest } from 'src/core/server'; import { LegacyAPICaller } from 'kibana/server'; import { Logger } from 'kibana/server'; +import { Logger as Logger_2 } from 'src/core/server'; import { LoggerFactory } from '@kbn/logging'; import { Moment } from 'moment'; import moment from 'moment'; @@ -658,7 +659,7 @@ export const indexPatterns: { // // @public (undocumented) export class IndexPatternsFetcher { - constructor(callDataCluster: LegacyAPICaller); + constructor(elasticsearchClient: ElasticsearchClient, allowNoIndices?: boolean); getFieldsForTimePattern(options: { pattern: string; metaFields: string[]; @@ -669,7 +670,7 @@ export class IndexPatternsFetcher { pattern: string | string[]; metaFields?: string[]; fieldCapsOptions?: { - allowNoIndices: boolean; + allow_no_indices: boolean; }; }): Promise; } diff --git a/src/plugins/discover/public/application/angular/context_app.html b/src/plugins/discover/public/application/angular/context_app.html index d609a497c4ba1e..ef7bc092061766 100644 --- a/src/plugins/discover/public/application/angular/context_app.html +++ b/src/plugins/discover/public/application/angular/context_app.html @@ -10,7 +10,6 @@ > - - - - - - - + predecessor-count="contextApp.state.queryParameters.predecessorCount" + predecessor-available="contextApp.state.rows.predecessors.length" + predecessor-status="contextApp.state.loadingStatus.predecessors.status" + on-change-predecessor-count="contextApp.actions.fetchGivenPredecessorRows" + successor-count="contextApp.state.queryParameters.successorCount" + successor-available="contextApp.state.rows.successors.length" + successor-status="contextApp.state.loadingStatus.successors.status" + on-change-successor-count="contextApp.actions.fetchGivenSuccessorRows" + > diff --git a/src/plugins/discover/public/application/components/context_app/__snapshots__/context_app_legacy.test.tsx.snap b/src/plugins/discover/public/application/components/context_app/__snapshots__/context_app_legacy.test.tsx.snap deleted file mode 100644 index 58305ee23cb210..00000000000000 --- a/src/plugins/discover/public/application/components/context_app/__snapshots__/context_app_legacy.test.tsx.snap +++ /dev/null @@ -1,741 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`ContextAppLegacy test renders correctly 1`] = ` - - - - - -
-
- -
- -
-
- - - - - -`; - -exports[`ContextAppLegacy test renders loading indicator 1`] = ` - - - - - -
- -
- -
- - Loading... - -
- -
- -
- - - - - -`; diff --git a/src/plugins/discover/public/application/components/context_app/context_app_legacy.test.tsx b/src/plugins/discover/public/application/components/context_app/context_app_legacy.test.tsx index 16d8cd78004f92..25576a90729447 100644 --- a/src/plugins/discover/public/application/components/context_app/context_app_legacy.test.tsx +++ b/src/plugins/discover/public/application/components/context_app/context_app_legacy.test.tsx @@ -23,6 +23,7 @@ import { IIndexPattern } from '../../../../../data/common/index_patterns'; import { mountWithIntl } from 'test_utils/enzyme_helpers'; import { DocTableLegacy } from '../../angular/doc_table/create_doc_table_react'; import { findTestSubject } from '@elastic/eui/lib/test'; +import { ActionBar } from '../../angular/context/components/action_bar/action_bar'; describe('ContextAppLegacy test', () => { const hit = { @@ -48,28 +49,36 @@ describe('ContextAppLegacy test', () => { columns: ['_source'], filter: () => {}, hits: [hit], - infiniteScroll: true, sorting: ['order_date', 'desc'], minimumVisibleRows: 5, indexPattern, status: 'loaded', + defaultStepSize: 5, + predecessorCount: 10, + successorCount: 10, + predecessorAvailable: 10, + successorAvailable: 10, + onChangePredecessorCount: jest.fn(), + onChangeSuccessorCount: jest.fn(), + predecessorStatus: 'loaded', + successorStatus: 'loaded', }; it('renders correctly', () => { const component = mountWithIntl(); - expect(component).toMatchSnapshot(); expect(component.find(DocTableLegacy).length).toBe(1); const loadingIndicator = findTestSubject(component, 'contextApp_loadingIndicator'); expect(loadingIndicator.length).toBe(0); + expect(component.find(ActionBar).length).toBe(2); }); it('renders loading indicator', () => { const props = { ...defaultProps }; props.status = 'loading'; const component = mountWithIntl(); - expect(component).toMatchSnapshot(); expect(component.find('DocTableLegacy').length).toBe(0); const loadingIndicator = findTestSubject(component, 'contextApp_loadingIndicator'); expect(loadingIndicator.length).toBe(1); + expect(component.find(ActionBar).length).toBe(2); }); }); diff --git a/src/plugins/discover/public/application/components/context_app/context_app_legacy.tsx b/src/plugins/discover/public/application/components/context_app/context_app_legacy.tsx index ee8b2f590f71c0..afb4a9a981e214 100644 --- a/src/plugins/discover/public/application/components/context_app/context_app_legacy.tsx +++ b/src/plugins/discover/public/application/components/context_app/context_app_legacy.tsx @@ -17,15 +17,15 @@ * under the License. */ import React from 'react'; -import { FormattedMessage } from '@kbn/i18n/react'; +import { FormattedMessage, I18nProvider } from '@kbn/i18n/react'; import { EuiPanel, EuiText } from '@elastic/eui'; -import { I18nProvider } from '@kbn/i18n/react'; import { DocTableLegacy, DocTableLegacyProps, } from '../../angular/doc_table/create_doc_table_react'; import { IIndexPattern, IndexPatternField } from '../../../../../data/common/index_patterns'; import { LOADING_STATUS } from './constants'; +import { ActionBar, ActionBarProps } from '../../angular/context/components/action_bar/action_bar'; export interface ContextAppProps { columns: string[]; @@ -35,15 +35,64 @@ export interface ContextAppProps { minimumVisibleRows: number; sorting: string[]; status: string; + defaultStepSize: number; + predecessorCount: number; + successorCount: number; + predecessorAvailable: number; + successorAvailable: number; + onChangePredecessorCount: (count: number) => void; + onChangeSuccessorCount: (count: number) => void; + predecessorStatus: string; + successorStatus: string; +} + +const PREDECESSOR_TYPE = 'predecessors'; +const SUCCESSOR_TYPE = 'successors'; + +function isLoading(status: string) { + return status !== LOADING_STATUS.LOADED && status !== LOADING_STATUS.FAILED; } export function ContextAppLegacy(renderProps: ContextAppProps) { - const { hits, filter, sorting, status } = renderProps; - const props = ({ ...renderProps } as unknown) as DocTableLegacyProps; - props.rows = hits; - props.onFilter = filter; - props.sort = sorting.map((el) => [el]); + const status = renderProps.status; const isLoaded = status === LOADING_STATUS.LOADED; + + const actionBarProps = (type: string) => { + const { + defaultStepSize, + successorCount, + predecessorCount, + predecessorAvailable, + successorAvailable, + predecessorStatus, + successorStatus, + onChangePredecessorCount, + onChangeSuccessorCount, + } = renderProps; + const isPredecessorType = type === PREDECESSOR_TYPE; + return { + defaultStepSize, + docCount: isPredecessorType ? predecessorCount : successorCount, + docCountAvailable: isPredecessorType ? predecessorAvailable : successorAvailable, + onChangeCount: isPredecessorType ? onChangePredecessorCount : onChangeSuccessorCount, + isLoading: isPredecessorType ? isLoading(predecessorStatus) : isLoading(successorStatus), + type, + isDisabled: !isLoaded, + } as ActionBarProps; + }; + + const docTableProps = () => { + const { hits, filter, sorting, columns, indexPattern, minimumVisibleRows } = renderProps; + return { + columns, + indexPattern, + minimumVisibleRows, + rows: hits, + onFilter: filter, + sort: sorting.map((el) => [el]), + } as DocTableLegacyProps; + }; + const loadingFeedback = () => { if (status === LOADING_STATUS.UNINITIALIZED || status === LOADING_STATUS.LOADING) { return ( @@ -59,17 +108,20 @@ export function ContextAppLegacy(renderProps: ContextAppProps) { } return null; }; + return ( + {loadingFeedback()} {isLoaded ? (
- +
) : null} + ); diff --git a/src/plugins/discover/public/application/components/context_app/context_app_legacy_directive.ts b/src/plugins/discover/public/application/components/context_app/context_app_legacy_directive.ts index af94c5537da288..4a315be513a0dc 100644 --- a/src/plugins/discover/public/application/components/context_app/context_app_legacy_directive.ts +++ b/src/plugins/discover/public/application/components/context_app/context_app_legacy_directive.ts @@ -25,8 +25,16 @@ export function createContextAppLegacy(reactDirective: any) { ['indexPattern', { watchDepth: 'reference' }], ['sorting', { watchDepth: 'reference' }], ['columns', { watchDepth: 'collection' }], - ['infiniteScroll', { watchDepth: 'reference' }], ['minimumVisibleRows', { watchDepth: 'reference' }], ['status', { watchDepth: 'reference' }], + ['defaultStepSize', { watchDepth: 'reference' }], + ['predecessorCount', { watchDepth: 'reference' }], + ['predecessorAvailable', { watchDepth: 'reference' }], + ['predecessorStatus', { watchDepth: 'reference' }], + ['onChangePredecessorCount', { watchDepth: 'reference' }], + ['successorCount', { watchDepth: 'reference' }], + ['successorAvailable', { watchDepth: 'reference' }], + ['successorStatus', { watchDepth: 'reference' }], + ['onChangeSuccessorCount', { watchDepth: 'reference' }], ]); } diff --git a/src/plugins/embeddable/public/lib/panel/_embeddable_panel.scss b/src/plugins/embeddable/public/lib/panel/_embeddable_panel.scss index cdc0f9f0e04513..6b8654f6c3528b 100644 --- a/src/plugins/embeddable/public/lib/panel/_embeddable_panel.scss +++ b/src/plugins/embeddable/public/lib/panel/_embeddable_panel.scss @@ -64,12 +64,12 @@ .embPanel__titleText { @include euiTextTruncate; + font-weight: $euiFontWeightBold; } .embPanel__placeholderTitleText { - @include euiTextTruncate; - font-weight: $euiFontWeightRegular; color: $euiColorMediumShade; + font-weight: $euiFontWeightRegular; } } diff --git a/src/plugins/embeddable/public/lib/panel/embeddable_panel.test.tsx b/src/plugins/embeddable/public/lib/panel/embeddable_panel.test.tsx index fcf79c1d6b211c..c717e4370231eb 100644 --- a/src/plugins/embeddable/public/lib/panel/embeddable_panel.test.tsx +++ b/src/plugins/embeddable/public/lib/panel/embeddable_panel.test.tsx @@ -19,7 +19,7 @@ import React from 'react'; import { mount } from 'enzyme'; -import { nextTick } from 'test_utils/enzyme_helpers'; +import { mountWithIntl, nextTick } from 'test_utils/enzyme_helpers'; import { findTestSubject } from '@elastic/eui/lib/test'; import { I18nProvider } from '@kbn/i18n/react'; @@ -343,6 +343,88 @@ test('HelloWorldContainer in edit mode shows edit mode actions', async () => { // expect(action.length).toBe(1); }); +test('Panel title customize link does not exist in view mode', async () => { + const inspector = inspectorPluginMock.createStartContract(); + + const container = new HelloWorldContainer( + { id: '123', panels: {}, viewMode: ViewMode.VIEW, hidePanelTitles: false }, + { getEmbeddableFactory } as any + ); + + const embeddable = await container.addNewEmbeddable< + ContactCardEmbeddableInput, + ContactCardEmbeddableOutput, + ContactCardEmbeddable + >(CONTACT_CARD_EMBEDDABLE, { + firstName: 'Vayon', + lastName: 'Poole', + }); + + const component = mountWithIntl( + Promise.resolve([])} + getAllEmbeddableFactories={start.getEmbeddableFactories} + getEmbeddableFactory={start.getEmbeddableFactory} + notifications={{} as any} + overlays={{} as any} + application={applicationMock} + inspector={inspector} + SavedObjectFinder={() => null} + /> + ); + + const titleLink = findTestSubject(component, 'embeddablePanelTitleLink'); + expect(titleLink.length).toBe(0); +}); + +test('Runs customize panel action on title click when in edit mode', async () => { + const inspector = inspectorPluginMock.createStartContract(); + + const container = new HelloWorldContainer( + { id: '123', panels: {}, viewMode: ViewMode.EDIT, hidePanelTitles: false }, + { getEmbeddableFactory } as any + ); + + const embeddable = await container.addNewEmbeddable< + ContactCardEmbeddableInput, + ContactCardEmbeddableOutput, + ContactCardEmbeddable + >(CONTACT_CARD_EMBEDDABLE, { + firstName: 'Vayon', + lastName: 'Poole', + }); + + const component = mountWithIntl( + Promise.resolve([])} + getAllEmbeddableFactories={start.getEmbeddableFactories} + getEmbeddableFactory={start.getEmbeddableFactory} + notifications={{} as any} + overlays={{} as any} + application={applicationMock} + inspector={inspector} + SavedObjectFinder={() => null} + /> + ); + + const titleExecute = jest.fn(); + component.setState((s: any) => ({ + ...s, + universalActions: { + ...s.universalActions, + customizePanelTitle: { execute: titleExecute, isCompatible: jest.fn() }, + }, + })); + + const titleLink = findTestSubject(component, 'embeddablePanelTitleLink'); + expect(titleLink.length).toBe(1); + titleLink.simulate('click'); + await nextTick(); + expect(titleExecute).toHaveBeenCalledTimes(1); +}); + test('Updates when hidePanelTitles is toggled', async () => { const inspector = inspectorPluginMock.createStartContract(); diff --git a/src/plugins/embeddable/public/lib/panel/embeddable_panel.tsx b/src/plugins/embeddable/public/lib/panel/embeddable_panel.tsx index 137f8c24b1fae0..1cd48e85469fd7 100644 --- a/src/plugins/embeddable/public/lib/panel/embeddable_panel.tsx +++ b/src/plugins/embeddable/public/lib/panel/embeddable_panel.tsx @@ -76,6 +76,7 @@ interface Props { interface State { panels: EuiContextMenuPanelDescriptor[]; + universalActions: PanelUniversalActions; focusedPanelIndex?: string; viewMode: ViewMode; hidePanelTitle: boolean; @@ -86,6 +87,14 @@ interface State { error?: EmbeddableError; } +interface PanelUniversalActions { + customizePanelTitle: CustomizePanelTitleAction; + addPanel: AddPanelAction; + inspectPanel: InspectPanelAction; + removePanel: RemovePanelAction; + editPanel: EditPanelAction; +} + export class EmbeddablePanel extends React.Component { private embeddableRoot: React.RefObject; private parentSubscription?: Subscription; @@ -102,6 +111,7 @@ export class EmbeddablePanel extends React.Component { Boolean(embeddable.getInput()?.hidePanelTitles); this.state = { + universalActions: this.getUniversalActions(), panels: [], viewMode, hidePanelTitle, @@ -229,6 +239,7 @@ export class EmbeddablePanel extends React.Component { getActionContextMenuPanel={this.getActionContextMenuPanel} hidePanelTitle={this.state.hidePanelTitle} isViewMode={viewOnlyMode} + customizeTitle={this.state.universalActions.customizePanelTitle} closeContextMenu={this.state.closeContextMenu} title={title} badges={this.state.badges} @@ -267,17 +278,7 @@ export class EmbeddablePanel extends React.Component { } }; - private getActionContextMenuPanel = async () => { - let regularActions = await this.props.getActions(CONTEXT_MENU_TRIGGER, { - embeddable: this.props.embeddable, - }); - - const { disabledActions } = this.props.embeddable.getInput(); - if (disabledActions) { - const removeDisabledActions = removeById(disabledActions); - regularActions = regularActions.filter(removeDisabledActions); - } - + private getUniversalActions = (): PanelUniversalActions => { const createGetUserData = (overlays: OverlayStart) => async function getUserData(context: { embeddable: IEmbeddable }) { return new Promise<{ title: string | undefined; hideTitle?: boolean }>((resolve) => { @@ -299,27 +300,41 @@ export class EmbeddablePanel extends React.Component { }); }; - // These actions are exposed on the context menu for every embeddable, they bypass the trigger + // Universal actions are exposed on the context menu for every embeddable, they bypass the trigger // registry. - const extraActions: Array> = [ - new CustomizePanelTitleAction(createGetUserData(this.props.overlays)), - new AddPanelAction( + return { + customizePanelTitle: new CustomizePanelTitleAction(createGetUserData(this.props.overlays)), + addPanel: new AddPanelAction( this.props.getEmbeddableFactory, this.props.getAllEmbeddableFactories, this.props.overlays, this.props.notifications, this.props.SavedObjectFinder ), - new InspectPanelAction(this.props.inspector), - new RemovePanelAction(), - new EditPanelAction( + inspectPanel: new InspectPanelAction(this.props.inspector), + removePanel: new RemovePanelAction(), + editPanel: new EditPanelAction( this.props.getEmbeddableFactory, this.props.application, this.props.stateTransfer ), - ]; + }; + }; - const sortedActions = [...regularActions, ...extraActions].sort(sortByOrderField); + private getActionContextMenuPanel = async () => { + let regularActions = await this.props.getActions(CONTEXT_MENU_TRIGGER, { + embeddable: this.props.embeddable, + }); + + const { disabledActions } = this.props.embeddable.getInput(); + if (disabledActions) { + const removeDisabledActions = removeById(disabledActions); + regularActions = regularActions.filter(removeDisabledActions); + } + + const sortedActions = [...regularActions, ...Object.values(this.state.universalActions)].sort( + sortByOrderField + ); return await buildContextMenuForActions({ actions: sortedActions.map((action) => ({ diff --git a/src/plugins/embeddable/public/lib/panel/panel_header/panel_header.tsx b/src/plugins/embeddable/public/lib/panel/panel_header/panel_header.tsx index 9bcef051a93596..44f5c3df2709d2 100644 --- a/src/plugins/embeddable/public/lib/panel/panel_header/panel_header.tsx +++ b/src/plugins/embeddable/public/lib/panel/panel_header/panel_header.tsx @@ -24,6 +24,7 @@ import { EuiToolTip, EuiScreenReaderOnly, EuiNotificationBadge, + EuiLink, } from '@elastic/eui'; import classNames from 'classnames'; import React from 'react'; @@ -32,6 +33,7 @@ import { PanelOptionsMenu } from './panel_options_menu'; import { IEmbeddable } from '../../embeddables'; import { EmbeddableContext, panelBadgeTrigger, panelNotificationTrigger } from '../../triggers'; import { uiToReactComponent } from '../../../../../kibana_react/public'; +import { CustomizePanelTitleAction } from '.'; export interface PanelHeaderProps { title?: string; @@ -44,6 +46,7 @@ export interface PanelHeaderProps { embeddable: IEmbeddable; headerId?: string; showPlaceholderTitle?: boolean; + customizeTitle: CustomizePanelTitleAction; } function renderBadges(badges: Array>, embeddable: IEmbeddable) { @@ -129,6 +132,7 @@ export function PanelHeader({ notifications, embeddable, headerId, + customizeTitle, }: PanelHeaderProps) { const description = getViewDescription(embeddable); const showTitle = !hidePanelTitle && (!isViewMode || title); @@ -172,11 +176,35 @@ export function PanelHeader({ } const renderTitle = () => { - const titleComponent = showTitle ? ( - - {title || placeholderTitle} - - ) : undefined; + let titleComponent; + if (showTitle) { + titleComponent = isViewMode ? ( + + {title || placeholderTitle} + + ) : ( + customizeTitle.execute({ embeddable })} + > + {title || placeholderTitle} + + ); + } return description ? ( { test('creates default ExecutionContext', () => { const execution = createExecution(); expect(execution.context).toMatchObject({ - getInitialInput: expect.any(Function), + getSearchContext: expect.any(Function), variables: expect.any(Object), types: expect.any(Object), }); @@ -143,6 +145,7 @@ describe('Execution', () => { const execution = new Execution({ executor, expression, + params: {}, }); expect(execution.expression).toBe(expression); }); @@ -153,6 +156,7 @@ describe('Execution', () => { const execution = new Execution({ ast: parseExpression(expression), executor, + params: {}, }); expect(execution.expression).toBe(expression); }); @@ -619,7 +623,7 @@ describe('Execution', () => { const execution = new Execution({ executor, ast: parseExpression('add val=1 | throws | add val=3'), - debug: true, + params: { debug: true }, }); execution.start(0); await execution.result; @@ -637,7 +641,7 @@ describe('Execution', () => { const execution = new Execution({ executor, ast: parseExpression('add val=1 | throws | add val=3'), - debug: true, + params: { debug: true }, }); execution.start(0); await execution.result; @@ -658,7 +662,7 @@ describe('Execution', () => { const execution = new Execution({ executor, ast: parseExpression('add val=1 | throws | add val=3'), - debug: true, + params: { debug: true }, }); execution.start(0); await execution.result; diff --git a/src/plugins/expressions/common/execution/execution.ts b/src/plugins/expressions/common/execution/execution.ts index 69140453f486d7..2bcf441b14203e 100644 --- a/src/plugins/expressions/common/execution/execution.ts +++ b/src/plugins/expressions/common/execution/execution.ts @@ -19,7 +19,7 @@ import { i18n } from '@kbn/i18n'; import { keys, last, mapValues, reduce, zipObject } from 'lodash'; -import { Executor, ExpressionExecOptions } from '../executor'; +import { Executor } from '../executor'; import { createExecutionContainer, ExecutionContainer } from './container'; import { createError } from '../util'; import { Defer, now } from '../../../kibana_utils/common'; @@ -39,6 +39,7 @@ import { getType, ExpressionValue } from '../expression_types'; import { ArgumentType, ExpressionFunction } from '../expression_functions'; import { getByAlias } from '../util/get_by_alias'; import { ExecutionContract } from './execution_contract'; +import { ExpressionExecutionParams } from '../service'; const createAbortErrorValue = () => createError({ @@ -46,20 +47,11 @@ const createAbortErrorValue = () => name: 'AbortError', }); -export interface ExecutionParams< - ExtraContext extends Record = Record -> { +export interface ExecutionParams { executor: Executor; ast?: ExpressionAstExpression; expression?: string; - context?: ExtraContext; - - /** - * Whether to execute expression in *debug mode*. In *debug mode* inputs and - * outputs as well as all resolved arguments and time it took to execute each - * function are saved and are available for introspection. - */ - debug?: boolean; + params: ExpressionExecutionParams; } const createDefaultInspectorAdapters = (): DefaultInspectorAdapters => ({ @@ -68,11 +60,10 @@ const createDefaultInspectorAdapters = (): DefaultInspectorAdapters => ({ }); export class Execution< - ExtraContext extends Record = Record, Input = unknown, Output = unknown, - InspectorAdapters extends Adapters = ExtraContext['inspectorAdapters'] extends object - ? ExtraContext['inspectorAdapters'] + InspectorAdapters extends Adapters = ExpressionExecutionParams['inspectorAdapters'] extends object + ? ExpressionExecutionParams['inspectorAdapters'] : DefaultInspectorAdapters > { /** @@ -92,7 +83,7 @@ export class Execution< * Execution context - object that allows to do side-effects. Context is passed * to every function. */ - public readonly context: ExecutionContext & ExtraContext; + public readonly context: ExecutionContext; /** * AbortController to cancel this Execution. @@ -126,11 +117,10 @@ export class Execution< * can return to other plugins for their consumption. */ public readonly contract: ExecutionContract< - ExtraContext, Input, Output, InspectorAdapters - > = new ExecutionContract(this); + > = new ExecutionContract(this); public readonly expression: string; @@ -142,17 +132,17 @@ export class Execution< return this.context.inspectorAdapters; } - constructor(public readonly params: ExecutionParams) { - const { executor } = params; + constructor(public readonly execution: ExecutionParams) { + const { executor } = execution; - if (!params.ast && !params.expression) { + if (!execution.ast && !execution.expression) { throw new TypeError('Execution params should contain at least .ast or .expression key.'); - } else if (params.ast && params.expression) { + } else if (execution.ast && execution.expression) { throw new TypeError('Execution params cannot contain both .ast and .expression key.'); } - this.expression = params.expression || formatExpression(params.ast!); - const ast = params.ast || parseExpression(this.expression); + this.expression = execution.expression || formatExpression(execution.ast!); + const ast = execution.ast || parseExpression(this.expression); this.state = createExecutionContainer({ ...executor.state.get(), @@ -161,14 +151,13 @@ export class Execution< }); this.context = { - getInitialInput: () => this.input, - variables: {}, + getSearchContext: () => this.execution.params.searchContext || {}, + getSearchSessionId: () => execution.params.searchSessionId, + variables: execution.params.variables || {}, types: executor.getTypes(), abortSignal: this.abortController.signal, - ...(params.context || ({} as ExtraContext)), - inspectorAdapters: (params.context && params.context.inspectorAdapters - ? params.context.inspectorAdapters - : createDefaultInspectorAdapters()) as InspectorAdapters, + inspectorAdapters: execution.params.inspectorAdapters || createDefaultInspectorAdapters(), + ...(execution.params as any).extraContext, }; } @@ -249,10 +238,10 @@ export class Execution< // actually have a `then` function which would be treated as a `Promise`. const { resolvedArgs } = await this.race(this.resolveArgs(fn, input, fnArgs)); args = resolvedArgs; - timeStart = this.params.debug ? now() : 0; + timeStart = this.execution.params.debug ? now() : 0; const output = await this.race(this.invokeFunction(fn, input, resolvedArgs)); - if (this.params.debug) { + if (this.execution.params.debug) { const timeEnd: number = now(); (link as ExpressionAstFunction).debug = { success: true, @@ -267,11 +256,11 @@ export class Execution< if (getType(output) === 'error') return output; input = output; } catch (rawError) { - const timeEnd: number = this.params.debug ? now() : 0; + const timeEnd: number = this.execution.params.debug ? now() : 0; const error = createError(rawError) as ExpressionValueError; error.error.message = `[${fnName}] > ${error.error.message}`; - if (this.params.debug) { + if (this.execution.params.debug) { (link as ExpressionAstFunction).debug = { success: false, fn: fn.name, @@ -404,9 +393,7 @@ export class Execution< const resolveArgFns = mapValues(argAstsWithDefaults, (asts, argName) => { return asts.map((item: ExpressionAstExpression) => { return async (subInput = input) => { - const output = await this.interpret(item, subInput, { - debug: this.params.debug, - }); + const output = await this.interpret(item, subInput); if (isExpressionValueError(output)) throw output.error; const casted = this.cast(output, argDefs[argName as any].types); return casted; @@ -438,17 +425,12 @@ export class Execution< return { resolvedArgs }; } - public async interpret( - ast: ExpressionAstNode, - input: T, - options?: ExpressionExecOptions - ): Promise { + public async interpret(ast: ExpressionAstNode, input: T): Promise { switch (getType(ast)) { case 'expression': - const execution = this.params.executor.createExecution( + const execution = this.execution.executor.createExecution( ast as ExpressionAstExpression, - this.context, - options + this.execution.params ); execution.start(input); return await execution.result; diff --git a/src/plugins/expressions/common/execution/execution_contract.test.ts b/src/plugins/expressions/common/execution/execution_contract.test.ts index c33f8a1a0f36e0..856b22470d782b 100644 --- a/src/plugins/expressions/common/execution/execution_contract.test.ts +++ b/src/plugins/expressions/common/execution/execution_contract.test.ts @@ -30,7 +30,7 @@ const createExecution = ( const execution = new Execution({ executor, ast: parseExpression(expression), - context, + params: { ...context }, }); return execution; }; diff --git a/src/plugins/expressions/common/execution/execution_contract.ts b/src/plugins/expressions/common/execution/execution_contract.ts index 79bb4c58ab48d5..f05f1ded82799d 100644 --- a/src/plugins/expressions/common/execution/execution_contract.ts +++ b/src/plugins/expressions/common/execution/execution_contract.ts @@ -25,21 +25,14 @@ import { ExpressionAstExpression } from '../ast'; * `ExecutionContract` is a wrapper around `Execution` class. It provides the * same functionality but does not expose Expressions plugin internals. */ -export class ExecutionContract< - ExtraContext extends Record = Record, - Input = unknown, - Output = unknown, - InspectorAdapters = unknown -> { +export class ExecutionContract { public get isPending(): boolean { const state = this.execution.state.get().state; const finished = state === 'error' || state === 'result'; return !finished; } - constructor( - protected readonly execution: Execution - ) {} + constructor(protected readonly execution: Execution) {} /** * Cancel the execution of the expression. This will set abort signal diff --git a/src/plugins/expressions/common/execution/types.ts b/src/plugins/expressions/common/execution/types.ts index 7c26e586fb790a..50475c3bd94aeb 100644 --- a/src/plugins/expressions/common/execution/types.ts +++ b/src/plugins/expressions/common/execution/types.ts @@ -26,11 +26,11 @@ import { SavedObject, SavedObjectAttributes } from '../../../../core/public'; * `ExecutionContext` is an object available to all functions during a single execution; * it provides various methods to perform side-effects. */ -export interface ExecutionContext { +export interface ExecutionContext { /** - * Get initial input with which execution started. + * Get search context of the expression. */ - getInitialInput: () => Input; + getSearchContext: () => ExecutionContextSearch; /** * Context variables that can be consumed using `var` and `var_set` functions. @@ -55,7 +55,7 @@ export interface ExecutionContext string | undefined; /** * Allows to fetch saved objects from ElasticSearch. In browser `getSavedObject` diff --git a/src/plugins/expressions/common/executor/executor.ts b/src/plugins/expressions/common/executor/executor.ts index fd7f5808f03400..85b5589b593af0 100644 --- a/src/plugins/expressions/common/executor/executor.ts +++ b/src/plugins/expressions/common/executor/executor.ts @@ -33,6 +33,7 @@ import { functionSpecs } from '../expression_functions/specs'; import { getByAlias } from '../util'; import { SavedObjectReference } from '../../../../core/types'; import { PersistableState } from '../../../kibana_utils/common'; +import { ExpressionExecutionParams } from '../service'; export interface ExpressionExecOptions { /** @@ -166,43 +167,34 @@ export class Executor = Record = Record - >( + public async run( ast: string | ExpressionAstExpression, input: Input, - context?: ExtraContext, - options?: ExpressionExecOptions + params: ExpressionExecutionParams = {} ) { - const execution = this.createExecution(ast, context, options); + const execution = this.createExecution(ast, params); execution.start(input); return (await execution.result) as Output; } - public createExecution< - ExtraContext extends Record = Record, - Input = unknown, - Output = unknown - >( + public createExecution( ast: string | ExpressionAstExpression, - context: ExtraContext = {} as ExtraContext, - { debug }: ExpressionExecOptions = {} as ExpressionExecOptions - ): Execution { - const params: ExecutionParams = { + params: ExpressionExecutionParams = {} + ): Execution { + const executionParams: ExecutionParams = { executor: this, - context: { - ...this.context, - ...context, - } as Context & ExtraContext, - debug, + params: { + ...params, + // for canvas we are passing this in, + // canvas should be refactored to not pass any extra context in + extraContext: this.context, + } as any, }; - if (typeof ast === 'string') params.expression = ast; - else params.ast = ast; + if (typeof ast === 'string') executionParams.expression = ast; + else executionParams.ast = ast; - const execution = new Execution(params); + const execution = new Execution(executionParams); return execution; } diff --git a/src/plugins/expressions/common/expression_functions/specs/kibana.ts b/src/plugins/expressions/common/expression_functions/specs/kibana.ts index 2144a8aba2d19d..3ec4c23eab28d6 100644 --- a/src/plugins/expressions/common/expression_functions/specs/kibana.ts +++ b/src/plugins/expressions/common/expression_functions/specs/kibana.ts @@ -44,15 +44,15 @@ export const kibana: ExpressionFunctionKibana = { args: {}, - fn(input, _, { search = {} }) { + fn(input, _, { getSearchContext }) { const output: ExpressionValueSearchContext = { // TODO: This spread is left here for legacy reasons, possibly Lens uses it. // TODO: But it shouldn't be need. ...input, type: 'kibana_context', - query: [...toArray(search.query), ...toArray((input || {}).query)], - filters: [...(search.filters || []), ...((input || {}).filters || [])], - timeRange: search.timeRange || (input ? input.timeRange : undefined), + query: [...toArray(getSearchContext().query), ...toArray((input || {}).query)], + filters: [...(getSearchContext().filters || []), ...((input || {}).filters || [])], + timeRange: getSearchContext().timeRange || (input ? input.timeRange : undefined), }; return output; diff --git a/src/plugins/expressions/common/expression_functions/specs/tests/kibana.test.ts b/src/plugins/expressions/common/expression_functions/specs/tests/kibana.test.ts index e5bd53f63c91d5..e5c4b92de4fdbc 100644 --- a/src/plugins/expressions/common/expression_functions/specs/tests/kibana.test.ts +++ b/src/plugins/expressions/common/expression_functions/specs/tests/kibana.test.ts @@ -46,8 +46,8 @@ describe('interpreter/functions#kibana', () => { timeRange: { from: '2', to: '3' }, }; context = { - search, - getInitialInput: () => input, + getSearchContext: () => search, + getSearchSessionId: () => undefined, types: {}, variables: {}, abortSignal: {} as any, diff --git a/src/plugins/expressions/common/expression_functions/specs/tests/theme.test.ts b/src/plugins/expressions/common/expression_functions/specs/tests/theme.test.ts index 263409f0caca28..88511d4fd571e2 100644 --- a/src/plugins/expressions/common/expression_functions/specs/tests/theme.test.ts +++ b/src/plugins/expressions/common/expression_functions/specs/tests/theme.test.ts @@ -37,7 +37,8 @@ describe('expression_functions', () => { }; context = { - getInitialInput: () => {}, + getSearchContext: () => ({} as any), + getSearchSessionId: () => undefined, types: {}, variables: { theme: themeProps }, abortSignal: {} as any, diff --git a/src/plugins/expressions/common/expression_functions/specs/tests/var.test.ts b/src/plugins/expressions/common/expression_functions/specs/tests/var.test.ts index ccf49ec918d3dc..762f34e3f55666 100644 --- a/src/plugins/expressions/common/expression_functions/specs/tests/var.test.ts +++ b/src/plugins/expressions/common/expression_functions/specs/tests/var.test.ts @@ -31,7 +31,8 @@ describe('expression_functions', () => { beforeEach(() => { input = { timeRange: { from: '0', to: '1' } }; context = { - getInitialInput: () => input, + getSearchContext: () => input, + getSearchSessionId: () => undefined, types: {}, variables: { test: 1 }, abortSignal: {} as any, diff --git a/src/plugins/expressions/common/expression_functions/specs/tests/var_set.test.ts b/src/plugins/expressions/common/expression_functions/specs/tests/var_set.test.ts index b1ae44e6f899eb..365ae5b89baeae 100644 --- a/src/plugins/expressions/common/expression_functions/specs/tests/var_set.test.ts +++ b/src/plugins/expressions/common/expression_functions/specs/tests/var_set.test.ts @@ -32,7 +32,8 @@ describe('expression_functions', () => { beforeEach(() => { input = { timeRange: { from: '0', to: '1' } }; context = { - getInitialInput: () => input, + getSearchContext: () => input, + getSearchSessionId: () => undefined, types: {}, variables: { test: 1 }, abortSignal: {} as any, diff --git a/src/plugins/expressions/common/mocks.ts b/src/plugins/expressions/common/mocks.ts index 502d88ac955ae6..52f96953885cf5 100644 --- a/src/plugins/expressions/common/mocks.ts +++ b/src/plugins/expressions/common/mocks.ts @@ -23,7 +23,8 @@ export const createMockExecutionContext = extraContext: ExtraContext = {} as ExtraContext ): ExecutionContext & ExtraContext => { const executionContext: ExecutionContext = { - getInitialInput: jest.fn(), + getSearchContext: jest.fn(), + getSearchSessionId: jest.fn(), variables: {}, types: {}, abortSignal: { @@ -37,7 +38,6 @@ export const createMockExecutionContext = requests: {} as any, data: {} as any, }, - search: {}, }; return { diff --git a/src/plugins/expressions/common/service/expressions_services.ts b/src/plugins/expressions/common/service/expressions_services.ts index 3d0fb968e8a3a2..abbba433ab3caa 100644 --- a/src/plugins/expressions/common/service/expressions_services.ts +++ b/src/plugins/expressions/common/service/expressions_services.ts @@ -17,7 +17,7 @@ * under the License. */ -import { Executor, ExpressionExecOptions } from '../executor'; +import { Executor } from '../executor'; import { AnyExpressionRenderDefinition, ExpressionRendererRegistry } from '../expression_renderers'; import { ExpressionAstExpression } from '../ast'; import { ExecutionContract } from '../execution/execution_contract'; @@ -25,6 +25,8 @@ import { AnyExpressionTypeDefinition } from '../expression_types'; import { AnyExpressionFunctionDefinition } from '../expression_functions'; import { SavedObjectReference } from '../../../../core/types'; import { PersistableState } from '../../../kibana_utils/common'; +import { Adapters } from '../../../inspector/common/adapters'; +import { ExecutionContextSearch } from '../execution'; /** * The public contract that `ExpressionsService` provides to other plugins @@ -45,6 +47,23 @@ export type ExpressionsServiceSetup = Pick< | 'fork' >; +export interface ExpressionExecutionParams { + searchContext?: ExecutionContextSearch; + + variables?: Record; + + /** + * Whether to execute expression in *debug mode*. In *debug mode* inputs and + * outputs as well as all resolved arguments and time it took to execute each + * function are saved and are available for introspection. + */ + debug?: boolean; + + searchSessionId?: string; + + inspectorAdapters?: Adapters; +} + /** * The public contract that `ExpressionsService` provides to other plugins * in Kibana Platform in *start* life-cycle. @@ -98,11 +117,10 @@ export interface ExpressionsServiceStart { * expressions.run('...', null, { elasticsearchClient }); * ``` */ - run: = Record>( + run: ( ast: string | ExpressionAstExpression, input: Input, - context?: ExtraContext, - options?: ExpressionExecOptions + params?: ExpressionExecutionParams ) => Promise; /** @@ -110,17 +128,12 @@ export interface ExpressionsServiceStart { * instance that tracks the progress of the execution and can be used to * interact with the execution. */ - execute: < - Input = unknown, - Output = unknown, - ExtraContext extends Record = Record - >( + execute: ( ast: string | ExpressionAstExpression, // This any is for legacy reasons. input: Input, - context?: ExtraContext, - options?: ExpressionExecOptions - ) => ExecutionContract; + params?: ExpressionExecutionParams + ) => ExecutionContract; /** * Create a new instance of `ExpressionsService`. The new instance inherits @@ -214,8 +227,8 @@ export class ExpressionsService implements PersistableState AnyExpressionRenderDefinition) ): void => this.renderers.register(definition); - public readonly run: ExpressionsServiceStart['run'] = (ast, input, context, options) => - this.executor.run(ast, input, context, options); + public readonly run: ExpressionsServiceStart['run'] = (ast, input, params) => + this.executor.run(ast, input, params); public readonly getFunction: ExpressionsServiceStart['getFunction'] = (name) => this.executor.getFunction(name); @@ -246,8 +259,8 @@ export class ExpressionsService implements PersistableState => this.executor.getTypes(); - public readonly execute: ExpressionsServiceStart['execute'] = ((ast, input, context, options) => { - const execution = this.executor.createExecution(ast, context, options); + public readonly execute: ExpressionsServiceStart['execute'] = ((ast, input, params) => { + const execution = this.executor.createExecution(ast, params); execution.start(input); return execution.contract; }) as ExpressionsServiceStart['execute']; diff --git a/src/plugins/expressions/public/loader.ts b/src/plugins/expressions/public/loader.ts index aef4b73f86e34a..91c482621de36c 100644 --- a/src/plugins/expressions/public/loader.ts +++ b/src/plugins/expressions/public/loader.ts @@ -145,18 +145,13 @@ export class ExpressionLoader { this.execution.cancel(); } this.setParams(params); - this.execution = getExpressionsService().execute( - expression, - params.context, - { - search: params.searchContext, - variables: params.variables || {}, - inspectorAdapters: params.inspectorAdapters, - }, - { - debug: params.debug, - } - ); + this.execution = getExpressionsService().execute(expression, params.context, { + searchContext: params.searchContext, + variables: params.variables || {}, + inspectorAdapters: params.inspectorAdapters, + searchSessionId: params.searchSessionId, + debug: params.debug, + }); const prevDataHandler = this.execution; const data = await prevDataHandler.getData(); @@ -188,6 +183,9 @@ export class ExpressionLoader { if (params.variables && this.params) { this.params.variables = params.variables; } + if (params.searchSessionId && this.params) { + this.params.searchSessionId = params.searchSessionId; + } this.params.debug = Boolean(params.debug); this.params.inspectorAdapters = (params.inspectorAdapters || diff --git a/src/plugins/expressions/public/public.api.md b/src/plugins/expressions/public/public.api.md index 4739b9434bdaa1..fe95cf5eb0cdac 100644 --- a/src/plugins/expressions/public/public.api.md +++ b/src/plugins/expressions/public/public.api.md @@ -90,33 +90,32 @@ export type DatatableColumnType = '_source' | 'attachment' | 'boolean' | 'date' export type DatatableRow = Record; // Warning: (ae-forgotten-export) The symbol "Adapters" needs to be exported by the entry point index.d.ts +// Warning: (ae-forgotten-export) The symbol "ExpressionExecutionParams" needs to be exported by the entry point index.d.ts // Warning: (ae-forgotten-export) The symbol "DefaultInspectorAdapters" needs to be exported by the entry point index.d.ts // Warning: (ae-missing-release-tag) "Execution" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) -export class Execution = Record, Input = unknown, Output = unknown, InspectorAdapters extends Adapters = ExtraContext['inspectorAdapters'] extends object ? ExtraContext['inspectorAdapters'] : DefaultInspectorAdapters> { - constructor(params: ExecutionParams); +export class Execution { + constructor(execution: ExecutionParams); cancel(): void; // (undocumented) cast(value: any, toTypeNames?: string[]): any; - readonly context: ExecutionContext & ExtraContext; - readonly contract: ExecutionContract; + readonly context: ExecutionContext; + readonly contract: ExecutionContract; + // (undocumented) + readonly execution: ExecutionParams; // (undocumented) readonly expression: string; input: Input; // (undocumented) get inspectorAdapters(): InspectorAdapters; - // Warning: (ae-forgotten-export) The symbol "ExpressionExecOptions" needs to be exported by the entry point index.d.ts - // // (undocumented) - interpret(ast: ExpressionAstNode, input: T, options?: ExpressionExecOptions): Promise; + interpret(ast: ExpressionAstNode, input: T): Promise; // (undocumented) invokeChain(chainArr: ExpressionAstFunction[], input: unknown): Promise; // (undocumented) invokeFunction(fn: ExpressionFunction, input: unknown, args: Record): Promise; // (undocumented) - readonly params: ExecutionParams; - // (undocumented) resolveArgs(fnDef: ExpressionFunction, input: unknown, argAsts: any): Promise; // (undocumented) get result(): Promise; @@ -134,15 +133,15 @@ export type ExecutionContainer = StateContainer { +export interface ExecutionContext { abortSignal: AbortSignal; - getInitialInput: () => Input; // Warning: (ae-forgotten-export) The symbol "SavedObjectAttributes" needs to be exported by the entry point index.d.ts // Warning: (ae-forgotten-export) The symbol "SavedObject" needs to be exported by the entry point index.d.ts getSavedObject?: (type: string, id: string) => Promise>; - inspectorAdapters: InspectorAdapters; // Warning: (ae-forgotten-export) The symbol "ExecutionContextSearch" needs to be exported by the entry point index.d.ts - search?: ExecutionContextSearch; + getSearchContext: () => ExecutionContextSearch; + getSearchSessionId: () => string | undefined; + inspectorAdapters: InspectorAdapters; types: Record; variables: Record; } @@ -150,11 +149,11 @@ export interface ExecutionContext = Record, Input = unknown, Output = unknown, InspectorAdapters = unknown> { - constructor(execution: Execution); +export class ExecutionContract { + constructor(execution: Execution); cancel: () => void; // (undocumented) - protected readonly execution: Execution; + protected readonly execution: Execution; getAst: () => ExpressionAstExpression; getData: () => Promise; getExpression: () => string; @@ -166,16 +165,15 @@ export class ExecutionContract = Re // Warning: (ae-missing-release-tag) "ExecutionParams" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) -export interface ExecutionParams = Record> { +export interface ExecutionParams { // (undocumented) ast?: ExpressionAstExpression; // (undocumented) - context?: ExtraContext; - debug?: boolean; - // (undocumented) executor: Executor; // (undocumented) expression?: string; + // (undocumented) + params: ExpressionExecutionParams; } // Warning: (ae-missing-release-tag) "ExecutionState" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) @@ -198,7 +196,7 @@ export class Executor = Record; // (undocumented) - createExecution = Record, Input = unknown, Output = unknown>(ast: string | ExpressionAstExpression, context?: ExtraContext, { debug }?: ExpressionExecOptions): Execution; + createExecution(ast: string | ExpressionAstExpression, params?: ExpressionExecutionParams): Execution; // (undocumented) static createWithDefaults = Record>(state?: ExecutorState): Executor; // (undocumented) @@ -228,7 +226,7 @@ export class Executor = Record AnyExpressionFunctionDefinition)): void; // (undocumented) registerType(typeDefinition: AnyExpressionTypeDefinition | (() => AnyExpressionTypeDefinition)): void; - run = Record>(ast: string | ExpressionAstExpression, input: Input, context?: ExtraContext, options?: ExpressionExecOptions): Promise; + run(ast: string | ExpressionAstExpression, input: Input, params?: ExpressionExecutionParams): Promise; // (undocumented) readonly state: ExecutorContainer; // (undocumented) @@ -613,12 +611,12 @@ export type ExpressionsServiceSetup = Pick = Record>(ast: string | ExpressionAstExpression, input: Input, context?: ExtraContext, options?: ExpressionExecOptions) => ExecutionContract; + execute: (ast: string | ExpressionAstExpression, input: Input, params?: ExpressionExecutionParams) => ExecutionContract; fork: () => ExpressionsService; getFunction: (name: string) => ReturnType; getRenderer: (name: string) => ReturnType; getType: (name: string) => ReturnType; - run: = Record>(ast: string | ExpressionAstExpression, input: Input, context?: ExtraContext, options?: ExpressionExecOptions) => Promise; + run: (ast: string | ExpressionAstExpression, input: Input, params?: ExpressionExecutionParams) => Promise; } // Warning: (ae-missing-release-tag) "ExpressionsSetup" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) @@ -894,6 +892,8 @@ export interface IExpressionLoaderParams { // (undocumented) searchContext?: ExecutionContextSearch; // (undocumented) + searchSessionId?: string; + // (undocumented) uiState?: unknown; // (undocumented) variables?: Record; diff --git a/src/plugins/expressions/public/types/index.ts b/src/plugins/expressions/public/types/index.ts index 054c5ac3dc467c..1643b5734ef1a8 100644 --- a/src/plugins/expressions/public/types/index.ts +++ b/src/plugins/expressions/public/types/index.ts @@ -53,6 +53,7 @@ export interface IExpressionLoaderParams { uiState?: unknown; inspectorAdapters?: Adapters; onRenderError?: RenderErrorHandlerFnType; + searchSessionId?: string; } export interface ExpressionRenderError extends Error { diff --git a/src/plugins/expressions/server/server.api.md b/src/plugins/expressions/server/server.api.md index fcdfd5ef3246cf..d6925a027358c8 100644 --- a/src/plugins/expressions/server/server.api.md +++ b/src/plugins/expressions/server/server.api.md @@ -87,34 +87,33 @@ export type DatatableColumnType = '_source' | 'attachment' | 'boolean' | 'date' export type DatatableRow = Record; // Warning: (ae-forgotten-export) The symbol "Adapters" needs to be exported by the entry point index.d.ts +// Warning: (ae-forgotten-export) The symbol "ExpressionExecutionParams" needs to be exported by the entry point index.d.ts // Warning: (ae-forgotten-export) The symbol "DefaultInspectorAdapters" needs to be exported by the entry point index.d.ts // Warning: (ae-missing-release-tag) "Execution" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) -export class Execution = Record, Input = unknown, Output = unknown, InspectorAdapters extends Adapters = ExtraContext['inspectorAdapters'] extends object ? ExtraContext['inspectorAdapters'] : DefaultInspectorAdapters> { - constructor(params: ExecutionParams); +export class Execution { + constructor(execution: ExecutionParams); cancel(): void; // (undocumented) cast(value: any, toTypeNames?: string[]): any; - readonly context: ExecutionContext & ExtraContext; + readonly context: ExecutionContext; // Warning: (ae-forgotten-export) The symbol "ExecutionContract" needs to be exported by the entry point index.d.ts - readonly contract: ExecutionContract; + readonly contract: ExecutionContract; + // (undocumented) + readonly execution: ExecutionParams; // (undocumented) readonly expression: string; input: Input; // (undocumented) get inspectorAdapters(): InspectorAdapters; - // Warning: (ae-forgotten-export) The symbol "ExpressionExecOptions" needs to be exported by the entry point index.d.ts - // // (undocumented) - interpret(ast: ExpressionAstNode, input: T, options?: ExpressionExecOptions): Promise; + interpret(ast: ExpressionAstNode, input: T): Promise; // (undocumented) invokeChain(chainArr: ExpressionAstFunction[], input: unknown): Promise; // (undocumented) invokeFunction(fn: ExpressionFunction, input: unknown, args: Record): Promise; // (undocumented) - readonly params: ExecutionParams; - // (undocumented) resolveArgs(fnDef: ExpressionFunction, input: unknown, argAsts: any): Promise; // (undocumented) get result(): Promise; @@ -132,15 +131,15 @@ export type ExecutionContainer = StateContainer { +export interface ExecutionContext { abortSignal: AbortSignal; - getInitialInput: () => Input; // Warning: (ae-forgotten-export) The symbol "SavedObjectAttributes" needs to be exported by the entry point index.d.ts // Warning: (ae-forgotten-export) The symbol "SavedObject" needs to be exported by the entry point index.d.ts getSavedObject?: (type: string, id: string) => Promise>; - inspectorAdapters: InspectorAdapters; // Warning: (ae-forgotten-export) The symbol "ExecutionContextSearch" needs to be exported by the entry point index.d.ts - search?: ExecutionContextSearch; + getSearchContext: () => ExecutionContextSearch; + getSearchSessionId: () => string | undefined; + inspectorAdapters: InspectorAdapters; types: Record; variables: Record; } @@ -148,16 +147,15 @@ export interface ExecutionContext = Record> { +export interface ExecutionParams { // (undocumented) ast?: ExpressionAstExpression; // (undocumented) - context?: ExtraContext; - debug?: boolean; - // (undocumented) executor: Executor; // (undocumented) expression?: string; + // (undocumented) + params: ExpressionExecutionParams; } // Warning: (ae-missing-release-tag) "ExecutionState" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) @@ -180,7 +178,7 @@ export class Executor = Record; // (undocumented) - createExecution = Record, Input = unknown, Output = unknown>(ast: string | ExpressionAstExpression, context?: ExtraContext, { debug }?: ExpressionExecOptions): Execution; + createExecution(ast: string | ExpressionAstExpression, params?: ExpressionExecutionParams): Execution; // (undocumented) static createWithDefaults = Record>(state?: ExecutorState): Executor; // (undocumented) @@ -210,7 +208,7 @@ export class Executor = Record AnyExpressionFunctionDefinition)): void; // (undocumented) registerType(typeDefinition: AnyExpressionTypeDefinition | (() => AnyExpressionTypeDefinition)): void; - run = Record>(ast: string | ExpressionAstExpression, input: Input, context?: ExtraContext, options?: ExpressionExecOptions): Promise; + run(ast: string | ExpressionAstExpression, input: Input, params?: ExpressionExecutionParams): Promise; // (undocumented) readonly state: ExecutorContainer; // (undocumented) diff --git a/src/plugins/home/public/application/components/solutions_section/solutions_section.tsx b/src/plugins/home/public/application/components/solutions_section/solutions_section.tsx index 13b70383147eba..bbc7f28235d6ad 100644 --- a/src/plugins/home/public/application/components/solutions_section/solutions_section.tsx +++ b/src/plugins/home/public/application/components/solutions_section/solutions_section.tsx @@ -25,8 +25,8 @@ import { SolutionPanel } from './solution_panel'; import { FeatureCatalogueEntry, FeatureCatalogueSolution } from '../../../'; const sortByOrder = ( - { order: orderA = 0 }: FeatureCatalogueSolution, - { order: orderB = 0 }: FeatureCatalogueSolution + { order: orderA = 0 }: FeatureCatalogueSolution | FeatureCatalogueEntry, + { order: orderB = 0 }: FeatureCatalogueSolution | FeatureCatalogueEntry ) => orderA - orderB; interface Props { @@ -38,7 +38,9 @@ interface Props { export const SolutionsSection: FC = ({ addBasePath, solutions, directories }) => { // Separate Kibana from other solutions const kibana = solutions.find(({ id }) => id === 'kibana'); - const kibanaApps = directories.filter(({ solutionId }) => solutionId === 'kibana'); + const kibanaApps = directories + .filter(({ solutionId }) => solutionId === 'kibana') + .sort(sortByOrder); solutions = solutions.sort(sortByOrder).filter(({ id }) => id !== 'kibana'); return ( diff --git a/src/plugins/kibana_overview/public/plugin.ts b/src/plugins/kibana_overview/public/plugin.ts index 663058ad16ec86..1139002dc19b6a 100644 --- a/src/plugins/kibana_overview/public/plugin.ts +++ b/src/plugins/kibana_overview/public/plugin.ts @@ -109,7 +109,7 @@ export class KibanaOverviewPlugin defaultMessage: 'Search and find insights.', }), i18n.translate('kibanaOverview.kibana.appDescription3', { - defaultMessage: 'Design pixel-perfect reports.', + defaultMessage: 'Design pixel-perfect presentations.', }), i18n.translate('kibanaOverview.kibana.appDescription4', { defaultMessage: 'Plot geographic data.', diff --git a/src/plugins/kibana_usage_collection/server/collectors/csp/csp_collector.ts b/src/plugins/kibana_usage_collection/server/collectors/csp/csp_collector.ts index c45a83588ee441..8cd60e5477d02a 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/csp/csp_collector.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/csp/csp_collector.ts @@ -17,7 +17,7 @@ * under the License. */ -import { UsageCollectionSetup, CollectorOptions } from 'src/plugins/usage_collection/server'; +import { UsageCollectionSetup, UsageCollectorOptions } from 'src/plugins/usage_collection/server'; import { HttpServiceSetup, CspConfig } from '../../../../../core/server'; interface Usage { @@ -26,7 +26,7 @@ interface Usage { rulesChangedFromDefault: boolean; } -export function createCspCollector(http: HttpServiceSetup): CollectorOptions { +export function createCspCollector(http: HttpServiceSetup): UsageCollectorOptions { return { type: 'csp', isReady: () => true, diff --git a/src/plugins/telemetry/schema/legacy_plugins.json b/src/plugins/telemetry/schema/legacy_plugins.json new file mode 100644 index 00000000000000..1a7c0ccb15082f --- /dev/null +++ b/src/plugins/telemetry/schema/legacy_plugins.json @@ -0,0 +1,21 @@ +{ + "properties": { + "localization": { + "properties": { + "locale": { + "type": "keyword" + }, + "integrities": { + "properties": { + "DYNAMIC_KEY": { + "type": "text" + } + } + }, + "labelsCount": { + "type": "long" + } + } + } + } +} diff --git a/src/plugins/usage_collection/server/collector/collector.test.ts b/src/plugins/usage_collection/server/collector/collector.test.ts index 375fe4f7686c0e..875414fbeec481 100644 --- a/src/plugins/usage_collection/server/collector/collector.test.ts +++ b/src/plugins/usage_collection/server/collector/collector.test.ts @@ -122,6 +122,7 @@ describe('collector', () => { type: 'my_test_collector', isReady: () => false, fetch: () => fetchOutput, + schema: { testPass: { type: 'long' } }, }); expect(collector.formatForBulkUpload(fetchOutput)).toStrictEqual({ type: 'kibana_stats', diff --git a/src/plugins/usage_collection/server/collector/collector.ts b/src/plugins/usage_collection/server/collector/collector.ts index 11a709c037783c..951418d448cbdd 100644 --- a/src/plugins/usage_collection/server/collector/collector.ts +++ b/src/plugins/usage_collection/server/collector/collector.ts @@ -71,17 +71,27 @@ export interface CollectorFetchContext { } export interface CollectorOptions { + /** + * Unique string identifier for the collector + */ type: string; init?: Function; + /** + * Method to return `true`/`false` to confirm if the collector is ready for the `fetch` method to be called. + */ + isReady: () => Promise | boolean; + /** + * Schema definition of the output of the `fetch` method. + */ schema?: MakeSchemaFrom; fetch: (collectorFetchContext: CollectorFetchContext) => Promise | T; /* * A hook for allowing the fetched data payload to be organized into a typed * data model for internal bulk upload. See defaultFormatterForBulkUpload for * a generic example. + * @deprecated Used only by the Legacy Monitoring collection (to be removed in 8.0) */ formatForBulkUpload?: CollectorFormatForBulkUpload; - isReady: () => Promise | boolean; } export class Collector { diff --git a/src/plugins/usage_collection/server/collector/collector_set.test.ts b/src/plugins/usage_collection/server/collector/collector_set.test.ts index 45a3437777c5f1..359a2d214f991a 100644 --- a/src/plugins/usage_collection/server/collector/collector_set.test.ts +++ b/src/plugins/usage_collection/server/collector/collector_set.test.ts @@ -20,7 +20,7 @@ import { noop } from 'lodash'; import { Collector } from './collector'; import { CollectorSet } from './collector_set'; -import { UsageCollector } from './usage_collector'; +import { UsageCollector, UsageCollectorOptions } from './usage_collector'; import { elasticsearchServiceMock, loggingSystemMock, @@ -73,8 +73,9 @@ describe('CollectorSet', () => { // Even for Collector vs. UsageCollector new UsageCollector(logger, { type: 'test_duplicated', - fetch: () => 2, + fetch: () => ({ prop: 2 }), isReady: () => false, + schema: { prop: { type: 'long' } }, }) ) ).toThrowError(`Usage collector's type "test_duplicated" is duplicated.`); @@ -252,7 +253,12 @@ describe('CollectorSet', () => { }); describe('isUsageCollector', () => { - const collectorOptions = { type: 'MY_TEST_COLLECTOR', fetch: () => {}, isReady: () => true }; + const collectorOptions: UsageCollectorOptions = { + type: 'MY_TEST_COLLECTOR', + fetch: () => ({ test: 1 }), + isReady: () => true, + schema: { test: { type: 'long' } }, + }; it('returns true only for UsageCollector instances', () => { const collectors = new CollectorSet({ logger }); diff --git a/src/plugins/usage_collection/server/collector/collector_set.ts b/src/plugins/usage_collection/server/collector/collector_set.ts index 4e64cbc1bf30f5..c52830cda6513b 100644 --- a/src/plugins/usage_collection/server/collector/collector_set.ts +++ b/src/plugins/usage_collection/server/collector/collector_set.ts @@ -26,7 +26,7 @@ import { SavedObjectsClientContract, } from 'kibana/server'; import { Collector, CollectorOptions } from './collector'; -import { UsageCollector } from './usage_collector'; +import { UsageCollector, UsageCollectorOptions } from './usage_collector'; interface CollectorSetConfig { logger: Logger; @@ -45,10 +45,22 @@ export class CollectorSet { this.maximumWaitTimeForAllCollectorsInS = maximumWaitTimeForAllCollectorsInS || 60; } - public makeStatsCollector = (options: CollectorOptions) => { + public makeStatsCollector = < + T, + U, + O extends CollectorOptions = CollectorOptions // Used to allow extra properties (the Collector constructor extends the class with the additional options provided) + >( + options: O + ) => { return new Collector(this.logger, options); }; - public makeUsageCollector = (options: CollectorOptions) => { + public makeUsageCollector = < + T, + U = T, + O extends UsageCollectorOptions = UsageCollectorOptions + >( + options: O + ) => { return new UsageCollector(this.logger, options); }; diff --git a/src/plugins/usage_collection/server/collector/index.ts b/src/plugins/usage_collection/server/collector/index.ts index c294ba77d3cdb0..da85f9ab181c98 100644 --- a/src/plugins/usage_collection/server/collector/index.ts +++ b/src/plugins/usage_collection/server/collector/index.ts @@ -26,4 +26,4 @@ export { CollectorOptions, CollectorFetchContext, } from './collector'; -export { UsageCollector } from './usage_collector'; +export { UsageCollector, UsageCollectorOptions } from './usage_collector'; diff --git a/src/plugins/usage_collection/server/collector/usage_collector.ts b/src/plugins/usage_collection/server/collector/usage_collector.ts index bf861a94fccfff..5bfc36537e0b05 100644 --- a/src/plugins/usage_collection/server/collector/usage_collector.ts +++ b/src/plugins/usage_collection/server/collector/usage_collector.ts @@ -17,13 +17,22 @@ * under the License. */ +import { Logger } from 'src/core/server'; import { KIBANA_STATS_TYPE } from '../../common/constants'; -import { Collector } from './collector'; +import { Collector, CollectorOptions } from './collector'; + +// Enforce the `schema` property for UsageCollectors +export type UsageCollectorOptions = CollectorOptions & + Required, 'schema'>>; export class UsageCollector extends Collector< T, U > { + constructor(protected readonly log: Logger, collectorOptions: UsageCollectorOptions) { + super(log, collectorOptions); + } + protected defaultFormatterForBulkUpload(result: T) { return { type: KIBANA_STATS_TYPE, diff --git a/src/plugins/usage_collection/server/index.ts b/src/plugins/usage_collection/server/index.ts index 80e34b1502cdab..f7a08fdb5e9dd8 100644 --- a/src/plugins/usage_collection/server/index.ts +++ b/src/plugins/usage_collection/server/index.ts @@ -25,6 +25,7 @@ export { MakeSchemaFrom, SchemaField, CollectorOptions, + UsageCollectorOptions, Collector, CollectorFetchContext, } from './collector'; diff --git a/src/plugins/vis_default_editor/public/components/agg_params_helper.test.ts b/src/plugins/vis_default_editor/public/components/agg_params_helper.test.ts index 834ad8b70ad0db..a56155db02f6b2 100644 --- a/src/plugins/vis_default_editor/public/components/agg_params_helper.test.ts +++ b/src/plugins/vis_default_editor/public/components/agg_params_helper.test.ts @@ -33,6 +33,7 @@ import { FieldParamEditor, OrderByParamEditor } from './controls'; import { EditorConfig } from './utils'; import { Schema } from '../schemas'; import { EditorVisState } from './sidebar/state/reducers'; +import { groupAndSortBy } from '../utils'; jest.mock('../utils', () => ({ groupAndSortBy: jest.fn(() => ['indexedFields']), @@ -169,6 +170,9 @@ describe('DefaultEditorAggParams helpers', () => { ], advanced: [], }); + + // Should be grouped using displayName as label + expect(groupAndSortBy).toHaveBeenCalledWith(expect.anything(), 'type', 'displayName', 'name'); }); }); diff --git a/src/plugins/vis_default_editor/public/components/agg_params_helper.ts b/src/plugins/vis_default_editor/public/components/agg_params_helper.ts index b13ca32601aa95..271fc75a0853ee 100644 --- a/src/plugins/vis_default_editor/public/components/agg_params_helper.ts +++ b/src/plugins/vis_default_editor/public/components/agg_params_helper.ts @@ -93,7 +93,7 @@ function getAggParamsToRender({ } } fields = filterAggTypeFields(availableFields, agg); - indexedFields = groupAndSortBy(fields, 'type', 'name'); + indexedFields = groupAndSortBy(fields, 'type', 'displayName', 'name'); if (fields && !indexedFields.length && index > 0) { // don't draw the rest of the options if there are no indexed fields and it's an extra param (index > 0). diff --git a/src/plugins/vis_default_editor/public/components/controls/field.tsx b/src/plugins/vis_default_editor/public/components/controls/field.tsx index bfc4f881f84588..41d6db25da5e2a 100644 --- a/src/plugins/vis_default_editor/public/components/controls/field.tsx +++ b/src/plugins/vis_default_editor/public/components/controls/field.tsx @@ -52,7 +52,7 @@ function FieldParamEditor({ }: FieldParamEditorProps) { const [isDirty, setIsDirty] = useState(false); const selectedOptions: ComboBoxGroupedOptions = value - ? [{ label: value.displayName || value.name, target: value }] + ? [{ label: value.displayName, target: value, key: value.name }] : []; const onChange = (options: EuiComboBoxOptionOption[]) => { diff --git a/src/plugins/vis_default_editor/public/utils.ts b/src/plugins/vis_default_editor/public/utils.ts index d0a9c067e9da28..11b7c07acc2c29 100644 --- a/src/plugins/vis_default_editor/public/utils.ts +++ b/src/plugins/vis_default_editor/public/utils.ts @@ -18,10 +18,12 @@ */ interface ComboBoxOption { + key?: string; label: string; target: T; } interface ComboBoxGroupedOption { + key?: string; label: string; options: Array>; } @@ -40,15 +42,22 @@ export type ComboBoxGroupedOptions = Array>; * @returns An array of grouped and sorted alphabetically `objects` that are compatible with EuiComboBox options. */ export function groupAndSortBy< - T extends Record, + T extends Record, TGroupBy extends string = 'type', - TLabelName extends string = 'title' ->(objects: T[], groupBy: TGroupBy, labelName: TLabelName): ComboBoxGroupedOptions { + TLabelName extends string = 'title', + TKeyName extends string = never +>( + objects: T[], + groupBy: TGroupBy, + labelName: TLabelName, + keyName?: TKeyName +): ComboBoxGroupedOptions { const groupedOptions = objects.reduce((array, obj) => { const group = array.find((element) => element.label === obj[groupBy]); const option = { label: obj[labelName], target: obj, + ...(keyName ? { key: obj[keyName] } : {}), }; if (group && group.options) { diff --git a/src/plugins/vis_type_timeseries/server/lib/get_fields.ts b/src/plugins/vis_type_timeseries/server/lib/get_fields.ts index 682d0a071e50de..b52188129f77f1 100644 --- a/src/plugins/vis_type_timeseries/server/lib/get_fields.ts +++ b/src/plugins/vis_type_timeseries/server/lib/get_fields.ts @@ -45,7 +45,7 @@ export async function getFields( payload: {}, pre: { indexPatternsService: new IndexPatternsFetcher( - requestContext.core.elasticsearch.legacy.client.callAsCurrentUser + requestContext.core.elasticsearch.client.asCurrentUser ), }, getUiSettingsService: () => requestContext.core.uiSettings.client, diff --git a/src/plugins/vis_type_timeseries/server/lib/search_strategies/strategies/abstract_search_strategy.test.js b/src/plugins/vis_type_timeseries/server/lib/search_strategies/strategies/abstract_search_strategy.test.js index ceae784cf74a64..613f33a47f1f45 100644 --- a/src/plugins/vis_type_timeseries/server/lib/search_strategies/strategies/abstract_search_strategy.test.js +++ b/src/plugins/vis_type_timeseries/server/lib/search_strategies/strategies/abstract_search_strategy.test.js @@ -50,7 +50,7 @@ describe('AbstractSearchStrategy', () => { expect(fields).toBe(mockedFields); expect(req.pre.indexPatternsService.getFieldsForWildcard).toHaveBeenCalledWith({ pattern: indexPattern, - fieldCapsOptions: { allowNoIndices: true }, + fieldCapsOptions: { allow_no_indices: true }, }); }); diff --git a/src/plugins/vis_type_timeseries/server/lib/search_strategies/strategies/abstract_search_strategy.ts b/src/plugins/vis_type_timeseries/server/lib/search_strategies/strategies/abstract_search_strategy.ts index 7b62ad310a354e..8b16048f0dce02 100644 --- a/src/plugins/vis_type_timeseries/server/lib/search_strategies/strategies/abstract_search_strategy.ts +++ b/src/plugins/vis_type_timeseries/server/lib/search_strategies/strategies/abstract_search_strategy.ts @@ -86,7 +86,7 @@ export class AbstractSearchStrategy { return await indexPatternsService!.getFieldsForWildcard({ pattern: indexPattern, - fieldCapsOptions: { allowNoIndices: true }, + fieldCapsOptions: { allow_no_indices: true }, }); } diff --git a/src/plugins/vis_type_vega/public/vega_fn.ts b/src/plugins/vis_type_vega/public/vega_fn.ts index c109bb3c6e90ca..c88b78948133c1 100644 --- a/src/plugins/vis_type_vega/public/vega_fn.ts +++ b/src/plugins/vis_type_vega/public/vega_fn.ts @@ -53,7 +53,7 @@ export const createVegaFn = ( Input, Arguments, Output, - ExecutionContext + ExecutionContext > => ({ name: 'vega', type: 'render', diff --git a/test/functional/apps/discover/_discover_histogram.ts b/test/functional/apps/discover/_discover_histogram.ts index e06783174e83b5..672becca614c97 100644 --- a/test/functional/apps/discover/_discover_histogram.ts +++ b/test/functional/apps/discover/_discover_histogram.ts @@ -31,7 +31,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { 'dateFormat:tz': 'Europe/Berlin', }; - describe('discover histogram', function describeIndexTests() { + // FLAKY: https://github.com/elastic/kibana/issues/81576 + describe.skip('discover histogram', function describeIndexTests() { before(async () => { await esArchiver.loadIfNeeded('logstash_functional'); await esArchiver.load('long_window_logstash'); diff --git a/test/functional/apps/discover/_doc_navigation.js b/test/functional/apps/discover/_doc_navigation.js index 87a150c7d6961c..31aef96918ffab 100644 --- a/test/functional/apps/discover/_doc_navigation.js +++ b/test/functional/apps/discover/_doc_navigation.js @@ -28,7 +28,8 @@ export default function ({ getService, getPageObjects }) { const esArchiver = getService('esArchiver'); const retry = getService('retry'); - describe('doc link in discover', function contextSize() { + // FLAKY: https://github.com/elastic/kibana/issues/78373 + describe.skip('doc link in discover', function contextSize() { beforeEach(async function () { log.debug('load kibana index with default index pattern'); await esArchiver.loadIfNeeded('discover'); diff --git a/test/functional/page_objects/common_page.ts b/test/functional/page_objects/common_page.ts index cc229ef0c2e08a..4a14d43aec2491 100644 --- a/test/functional/page_objects/common_page.ts +++ b/test/functional/page_objects/common_page.ts @@ -117,11 +117,12 @@ export function CommonPageProvider({ getService, getPageObjects }: FtrProviderCo } else { log.debug(`navigateToUrl ${appUrl}`); await browser.get(appUrl, insertTimestamp); - // accept alert if it pops up - const alert = await browser.getAlert(); - await alert?.accept(); } + // accept alert if it pops up + const alert = await browser.getAlert(); + await alert?.accept(); + const currentUrl = shouldLoginIfPrompted ? await this.loginIfPrompted(appUrl, insertTimestamp) : await browser.getCurrentUrl(); diff --git a/test/functional/page_objects/time_picker.ts b/test/functional/page_objects/time_picker.ts index 3ac6c83e61f145..272fc5a7914be7 100644 --- a/test/functional/page_objects/time_picker.ts +++ b/test/functional/page_objects/time_picker.ts @@ -35,12 +35,18 @@ export type CommonlyUsed = export function TimePickerProvider({ getService, getPageObjects }: FtrProviderContext) { const log = getService('log'); - const retry = getService('retry'); const find = getService('find'); const browser = getService('browser'); const testSubjects = getService('testSubjects'); const { header } = getPageObjects(['header']); const kibanaServer = getService('kibanaServer'); + const MenuToggle = getService('MenuToggle'); + + const quickSelectTimeMenuToggle = new MenuToggle({ + name: 'QuickSelectTime Menu', + menuTestSubject: 'superDatePickerQuickMenu', + toggleButtonTestSubject: 'superDatePickerToggleQuickMenuButton', + }); class TimePicker { defaultStartTime = 'Sep 19, 2015 @ 06:31:44.000'; @@ -158,34 +164,8 @@ export function TimePickerProvider({ getService, getPageObjects }: FtrProviderCo return await find.existsByCssSelector('.euiDatePickerRange--readOnly'); } - public async isQuickSelectMenuOpen() { - return await testSubjects.exists('superDatePickerQuickMenu'); - } - - public async openQuickSelectTimeMenu() { - log.debug('openQuickSelectTimeMenu'); - const isMenuOpen = await this.isQuickSelectMenuOpen(); - if (!isMenuOpen) { - log.debug('opening quick select menu'); - await retry.try(async () => { - await testSubjects.click('superDatePickerToggleQuickMenuButton'); - }); - } - } - - public async closeQuickSelectTimeMenu() { - log.debug('closeQuickSelectTimeMenu'); - const isMenuOpen = await this.isQuickSelectMenuOpen(); - if (isMenuOpen) { - log.debug('closing quick select menu'); - await retry.try(async () => { - await testSubjects.click('superDatePickerToggleQuickMenuButton'); - }); - } - } - public async getRefreshConfig(keepQuickSelectOpen = false) { - await this.openQuickSelectTimeMenu(); + await quickSelectTimeMenuToggle.open(); const interval = await testSubjects.getAttribute( 'superDatePickerRefreshIntervalInput', 'value' @@ -207,7 +187,7 @@ export function TimePickerProvider({ getService, getPageObjects }: FtrProviderCo 'superDatePickerToggleRefreshButton' ); if (!keepQuickSelectOpen) { - await this.closeQuickSelectTimeMenu(); + await quickSelectTimeMenuToggle.close(); } return { @@ -270,26 +250,26 @@ export function TimePickerProvider({ getService, getPageObjects }: FtrProviderCo } public async startAutoRefresh(intervalS = 3) { - await this.openQuickSelectTimeMenu(); + await quickSelectTimeMenuToggle.open(); await this.inputValue('superDatePickerRefreshIntervalInput', intervalS.toString()); const refreshConfig = await this.getRefreshConfig(true); if (refreshConfig.isPaused) { log.debug('start auto refresh'); await testSubjects.click('superDatePickerToggleRefreshButton'); } - await this.closeQuickSelectTimeMenu(); + await quickSelectTimeMenuToggle.close(); } public async pauseAutoRefresh() { log.debug('pauseAutoRefresh'); const refreshConfig = await this.getRefreshConfig(true); + if (!refreshConfig.isPaused) { log.debug('pause auto refresh'); await testSubjects.click('superDatePickerToggleRefreshButton'); - await this.closeQuickSelectTimeMenu(); } - await this.closeQuickSelectTimeMenu(); + await quickSelectTimeMenuToggle.close(); } public async resumeAutoRefresh() { @@ -300,7 +280,7 @@ export function TimePickerProvider({ getService, getPageObjects }: FtrProviderCo await testSubjects.click('superDatePickerToggleRefreshButton'); } - await this.closeQuickSelectTimeMenu(); + await quickSelectTimeMenuToggle.close(); } public async setHistoricalDataRange() { diff --git a/test/functional/services/common/test_subjects.ts b/test/functional/services/common/test_subjects.ts index 204edcb215107a..e464eaf9436277 100644 --- a/test/functional/services/common/test_subjects.ts +++ b/test/functional/services/common/test_subjects.ts @@ -170,11 +170,27 @@ export function TestSubjectsProvider({ getService }: FtrProviderContext) { public async getAttribute( selector: string, attribute: string, - timeout?: number + options?: + | number + | { + findTimeout?: number; + tryTimeout?: number; + } ): Promise { - return await retry.try(async () => { - log.debug(`TestSubjects.getAttribute(${selector}, ${attribute})`); - const element = await this.find(selector, timeout); + const findTimeout = + (typeof options === 'number' ? options : options?.findTimeout) ?? + config.get('timeouts.find'); + + const tryTimeout = + (typeof options !== 'number' ? options?.tryTimeout : undefined) ?? + config.get('timeouts.try'); + + log.debug( + `TestSubjects.getAttribute(${selector}, ${attribute}, tryTimeout=${tryTimeout}, findTimeout=${findTimeout})` + ); + + return await retry.tryForTime(tryTimeout, async () => { + const element = await this.find(selector, findTimeout); return await element.getAttribute(attribute); }); } diff --git a/test/functional/services/index.ts b/test/functional/services/index.ts index 057ae0bd13b6e7..2c71fd8ef8f55f 100644 --- a/test/functional/services/index.ts +++ b/test/functional/services/index.ts @@ -57,6 +57,7 @@ import { import { ListingTableProvider } from './listing_table'; import { SavedQueryManagementComponentProvider } from './saved_query_management_component'; import { KibanaSupertestProvider } from './supertest'; +import { MenuToggleProvider } from './menu_toggle'; export const services = { ...commonServiceProviders, @@ -93,4 +94,5 @@ export const services = { elasticChart: ElasticChartProvider, supertest: KibanaSupertestProvider, managementMenu: ManagementMenuProvider, + MenuToggle: MenuToggleProvider, }; diff --git a/test/functional/services/menu_toggle.ts b/test/functional/services/menu_toggle.ts new file mode 100644 index 00000000000000..073aa846f47a85 --- /dev/null +++ b/test/functional/services/menu_toggle.ts @@ -0,0 +1,77 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { FtrProviderContext } from '../ftr_provider_context'; + +export function MenuToggleProvider({ getService }: FtrProviderContext) { + const log = getService('log'); + const retry = getService('retry'); + const testSubjects = getService('testSubjects'); + + interface Options { + name: string; + menuTestSubject: string; + toggleButtonTestSubject: string; + } + + return class MenuToggle { + private readonly name: string; + private readonly menuTestSubject: string; + private readonly toggleButtonTestSubject: string; + + constructor(options: Options) { + this.name = options.name; + this.menuTestSubject = options.menuTestSubject; + this.toggleButtonTestSubject = options.toggleButtonTestSubject; + } + + async open() { + await this.setState(true); + } + + async close() { + await this.setState(false); + } + + private async setState(expectedState: boolean) { + log.debug( + `setting menu open state [name=${this.name}] [state=${expectedState ? 'open' : 'closed'}]` + ); + + await retry.try(async () => { + // if the menu is clearly in the expected state already, bail out quickly if so + const isOpen = await testSubjects.exists(this.menuTestSubject, { timeout: 1000 }); + if (isOpen === expectedState) { + return; + } + + // toggle the view state by clicking the button + await testSubjects.click(this.toggleButtonTestSubject); + + if (expectedState === true) { + // wait for up to 10 seconds for the menu to show up, otherwise fail and retry + await testSubjects.existOrFail(this.menuTestSubject, { timeout: 10000 }); + } else { + // wait for the form to hide, otherwise fail and retry + await testSubjects.waitForDeleted(this.menuTestSubject); + } + }); + } + }; +} diff --git a/test/interpreter_functional/plugins/kbn_tp_run_pipeline/public/app/components/main.tsx b/test/interpreter_functional/plugins/kbn_tp_run_pipeline/public/app/components/main.tsx index b4f9634b23d299..8c62fa246dd591 100644 --- a/test/interpreter_functional/plugins/kbn_tp_run_pipeline/public/app/components/main.tsx +++ b/test/interpreter_functional/plugins/kbn_tp_run_pipeline/public/app/components/main.tsx @@ -63,7 +63,7 @@ class Main extends React.Component<{}, State> { return getExpressions() .execute(expression, context || { type: 'null' }, { inspectorAdapters: adapters, - search: initialContext as any, + searchContext: initialContext as any, }) .getData(); }; diff --git a/x-pack/plugins/actions/server/saved_objects/migrations.test.ts b/x-pack/plugins/actions/server/saved_objects/migrations.test.ts index 1fa5889e77cb04..947d84fcfc638f 100644 --- a/x-pack/plugins/actions/server/saved_objects/migrations.test.ts +++ b/x-pack/plugins/actions/server/saved_objects/migrations.test.ts @@ -24,7 +24,11 @@ describe('7.10.0', () => { test('add hasAuth config property for .email actions', () => { const migration710 = getMigrations(encryptedSavedObjectsSetup)['7.10.0']; const action = getMockDataForEmail({}); - expect(migration710(action, context)).toMatchObject({ + const migratedAction = migration710(action, context); + expect(migratedAction.attributes.config).toEqual({ + hasAuth: true, + }); + expect(migratedAction).toEqual({ ...action, attributes: { ...action.attributes, @@ -38,7 +42,11 @@ describe('7.10.0', () => { test('rename cases configuration object', () => { const migration710 = getMigrations(encryptedSavedObjectsSetup)['7.10.0']; const action = getMockData({}); - expect(migration710(action, context)).toMatchObject({ + const migratedAction = migration710(action, context); + expect(migratedAction.attributes.config).toEqual({ + incidentConfiguration: { mapping: [] }, + }); + expect(migratedAction).toEqual({ ...action, attributes: { ...action.attributes, diff --git a/x-pack/plugins/actions/server/saved_objects/migrations.ts b/x-pack/plugins/actions/server/saved_objects/migrations.ts index 993beef8d9b2bd..35d30accecedb5 100644 --- a/x-pack/plugins/actions/server/saved_objects/migrations.ts +++ b/x-pack/plugins/actions/server/saved_objects/migrations.ts @@ -70,6 +70,9 @@ function renameCasesConfigurationObject( const addHasAuthConfigurationObject = ( doc: SavedObjectUnsanitizedDoc ): SavedObjectUnsanitizedDoc => { + if (doc.attributes.actionTypeId !== '.email') { + return doc; + } const hasAuth = !!doc.attributes.secrets.user || !!doc.attributes.secrets.password; return { ...doc, diff --git a/x-pack/plugins/alerts/server/task_runner/task_runner.test.ts b/x-pack/plugins/alerts/server/task_runner/task_runner.test.ts index d9af3d0ae6d5bd..8e345d6ff66a8a 100644 --- a/x-pack/plugins/alerts/server/task_runner/task_runner.test.ts +++ b/x-pack/plugins/alerts/server/task_runner/task_runner.test.ts @@ -621,9 +621,7 @@ describe('Task Runner', () => { expect(await taskRunner.run()).toMatchInlineSnapshot(` Object { "runAt": 1970-01-01T00:00:10.000Z, - "state": Object { - "previousStartedAt": 1970-01-01T00:00:00.000Z, - }, + "state": Object {}, } `); expect(taskRunnerFactoryInitializerParams.logger.error).toHaveBeenCalledWith( @@ -727,9 +725,7 @@ describe('Task Runner', () => { expect(runnerResult).toMatchInlineSnapshot(` Object { "runAt": 1970-01-01T00:00:10.000Z, - "state": Object { - "previousStartedAt": 1970-01-01T00:00:00.000Z, - }, + "state": Object {}, } `); @@ -781,9 +777,7 @@ describe('Task Runner', () => { expect(runnerResult).toMatchInlineSnapshot(` Object { "runAt": 1970-01-01T00:05:00.000Z, - "state": Object { - "previousStartedAt": 1970-01-01T00:00:00.000Z, - }, + "state": Object {}, } `); }); @@ -814,9 +808,7 @@ describe('Task Runner', () => { expect(runnerResult).toMatchInlineSnapshot(` Object { "runAt": 1970-01-01T00:05:00.000Z, - "state": Object { - "previousStartedAt": 1970-01-01T00:00:00.000Z, - }, + "state": Object {}, } `); }); @@ -846,13 +838,48 @@ describe('Task Runner', () => { expect(runnerResult).toMatchInlineSnapshot(` Object { "runAt": 1970-01-01T00:05:00.000Z, - "state": Object { - "previousStartedAt": 1970-01-01T00:00:00.000Z, - }, + "state": Object {}, } `); }); + test(`doesn't change previousStartedAt when it fails to run`, async () => { + const originalAlertSate = { + previousStartedAt: '1970-01-05T00:00:00.000Z', + }; + + alertType.executor.mockImplementation( + ({ services: executorServices }: AlertExecutorOptions) => { + throw new Error('OMG'); + } + ); + + const taskRunner = new TaskRunner( + alertType, + { + ...mockedTaskInstance, + state: originalAlertSate, + }, + taskRunnerFactoryInitializerParams + ); + + alertsClient.get.mockResolvedValueOnce(mockedAlertTypeSavedObject); + encryptedSavedObjectsClient.getDecryptedAsInternalUser.mockResolvedValueOnce({ + id: '1', + type: 'alert', + attributes: { + apiKey: Buffer.from('123:abc').toString('base64'), + }, + references: [], + }); + + const runnerResult = await taskRunner.run(); + + expect(runnerResult.state.previousStartedAt).toEqual( + new Date(originalAlertSate.previousStartedAt) + ); + }); + test('avoids rescheduling a failed Alert Task Runner when it throws due to failing to fetch the alert', async () => { alertsClient.get.mockImplementation(() => { throw SavedObjectsErrorHelpers.createGenericNotFoundError('task', '1'); @@ -878,9 +905,7 @@ describe('Task Runner', () => { expect(runnerResult).toMatchInlineSnapshot(` Object { "runAt": undefined, - "state": Object { - "previousStartedAt": 1970-01-01T00:00:00.000Z, - }, + "state": Object {}, } `); }); diff --git a/x-pack/plugins/alerts/server/task_runner/task_runner.ts b/x-pack/plugins/alerts/server/task_runner/task_runner.ts index 1ccf14a3a53340..7da16717aef9f0 100644 --- a/x-pack/plugins/alerts/server/task_runner/task_runner.ts +++ b/x-pack/plugins/alerts/server/task_runner/task_runner.ts @@ -322,7 +322,7 @@ export class TaskRunner { async run(): Promise { const { params: { alertId, spaceId }, - startedAt: previousStartedAt, + startedAt, state: originalState, } = this.taskInstance; @@ -360,7 +360,7 @@ export class TaskRunner { (stateUpdates: AlertTaskState) => { return { ...stateUpdates, - previousStartedAt, + previousStartedAt: startedAt, }; }, (err: Error) => { @@ -370,10 +370,7 @@ export class TaskRunner { } else { this.logger.error(message); } - return { - ...originalState, - previousStartedAt, - }; + return originalState; } ), runAt: resolveErr(runAt, (err) => { diff --git a/x-pack/plugins/apm/server/lib/index_pattern/get_dynamic_index_pattern.ts b/x-pack/plugins/apm/server/lib/index_pattern/get_dynamic_index_pattern.ts index 49030dc8cacc56..cf1f4852002ec8 100644 --- a/x-pack/plugins/apm/server/lib/index_pattern/get_dynamic_index_pattern.ts +++ b/x-pack/plugins/apm/server/lib/index_pattern/get_dynamic_index_pattern.ts @@ -5,7 +5,6 @@ */ import LRU from 'lru-cache'; -import { LegacyAPICaller } from '../../../../../../src/core/server'; import { IndexPatternsFetcher, FieldDescriptor, @@ -45,8 +44,7 @@ export const getDynamicIndexPattern = async ({ } const indexPatternsFetcher = new IndexPatternsFetcher( - (...rest: Parameters) => - context.core.elasticsearch.legacy.client.callAsCurrentUser(...rest) + context.core.elasticsearch.client.asCurrentUser ); // Since `getDynamicIndexPattern` is called in setup_request (and thus by every endpoint) diff --git a/x-pack/plugins/apm/typings/elasticsearch/aggregations.ts b/x-pack/plugins/apm/typings/elasticsearch/aggregations.ts index 8b3163e44915a3..a869c1e1186fa0 100644 --- a/x-pack/plugins/apm/typings/elasticsearch/aggregations.ts +++ b/x-pack/plugins/apm/typings/elasticsearch/aggregations.ts @@ -145,6 +145,15 @@ export interface AggregationOptionsByType { >; keyed?: boolean; } & AggregationSourceOptions; + range: { + field: string; + ranges: Array< + | { key?: string; from: string | number } + | { key?: string; to: string | number } + | { key?: string; from: string | number; to: string | number } + >; + keyed?: boolean; + }; auto_date_histogram: { buckets: number; } & AggregationSourceOptions; @@ -324,6 +333,18 @@ interface AggregationResponsePart< ? Record : { buckets: DateRangeBucket[] }; }; + range: { + buckets: TAggregationOptionsMap extends { range: { keyed: true } } + ? Record< + string, + DateRangeBucket & + SubAggregationResponseOf + > + : Array< + DateRangeBucket & + SubAggregationResponseOf + >; + }; auto_date_histogram: { buckets: Array< DateHistogramBucket & diff --git a/x-pack/plugins/canvas/public/feature_catalogue_entry.ts b/x-pack/plugins/canvas/public/feature_catalogue_entry.ts index ccc642177780d5..2b6d2f105454ba 100644 --- a/x-pack/plugins/canvas/public/feature_catalogue_entry.ts +++ b/x-pack/plugins/canvas/public/feature_catalogue_entry.ts @@ -11,7 +11,7 @@ export const featureCatalogueEntry = { id: 'canvas', title: 'Canvas', subtitle: i18n.translate('xpack.canvas.featureCatalogue.canvasSubtitle', { - defaultMessage: 'Design pixel-perfect reports.', + defaultMessage: 'Design pixel-perfect presentations.', }), description: i18n.translate('xpack.canvas.appDescription', { defaultMessage: 'Showcase your data in a pixel-perfect way.', diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/constants.ts b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/constants.ts index 53aa3db00b66ab..decf1e2158744a 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/constants.ts +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/constants.ts @@ -93,3 +93,5 @@ export const TOKEN_TYPE_INFO = [ ]; export const FLYOUT_ARIA_LABEL_ID = 'credentialsFlyoutTitle'; + +export const DOCS_HREF = 'https://www.elastic.co/guide/en/app-search/current/authentication.html'; diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/body.test.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/body.test.tsx index d2e7ff5f32dd47..e9217da1636364 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/body.test.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/body.test.tsx @@ -4,15 +4,98 @@ * you may not use this file except in compliance with the Elastic License. */ +import { setMockValues, setMockActions } from '../../../../__mocks__/kea.mock'; + import React from 'react'; import { shallow } from 'enzyme'; -import { EuiFlyoutBody } from '@elastic/eui'; +import { EuiFlyoutBody, EuiForm } from '@elastic/eui'; + +import { ApiTokenTypes } from '../constants'; +import { defaultApiToken } from '../credentials_logic'; +import { + FormKeyName, + FormKeyType, + FormKeyReadWriteAccess, + FormKeyEngineAccess, + FormKeyUpdateWarning, +} from './form_components'; import { CredentialsFlyoutBody } from './body'; describe('CredentialsFlyoutBody', () => { + const values = { + activeApiToken: defaultApiToken, + activeApiTokenExists: false, + }; + const actions = { + onApiTokenChange: jest.fn(), + }; + + beforeEach(() => { + jest.clearAllMocks(); + setMockValues(values); + setMockActions(actions); + }); + it('renders', () => { const wrapper = shallow(); + expect(wrapper.find(EuiFlyoutBody)).toHaveLength(1); + expect(wrapper.find(EuiForm)).toHaveLength(1); + }); + + it('shows the expected form components on default private key creation', () => { + const wrapper = shallow(); + + expect(wrapper.find(FormKeyName)).toHaveLength(1); + expect(wrapper.find(FormKeyType)).toHaveLength(1); + expect(wrapper.find(FormKeyReadWriteAccess)).toHaveLength(1); + expect(wrapper.find(FormKeyEngineAccess)).toHaveLength(1); + expect(wrapper.find(FormKeyUpdateWarning)).toHaveLength(0); + }); + + it('does not show read-write access options for search keys', () => { + setMockValues({ + ...values, + activeApiToken: { + ...defaultApiToken, + type: ApiTokenTypes.Search, + }, + }); + const wrapper = shallow(); + + expect(wrapper.find(FormKeyReadWriteAccess)).toHaveLength(0); + expect(wrapper.find(FormKeyEngineAccess)).toHaveLength(1); + }); + + it('does not show read-write or engine access options for admin keys', () => { + setMockValues({ + ...values, + activeApiToken: { + ...defaultApiToken, + type: ApiTokenTypes.Admin, + }, + }); + const wrapper = shallow(); + + expect(wrapper.find(FormKeyReadWriteAccess)).toHaveLength(0); + expect(wrapper.find(FormKeyEngineAccess)).toHaveLength(0); + }); + + it('shows a warning if updating an existing key', () => { + setMockValues({ ...values, activeApiTokenExists: true }); + const wrapper = shallow(); + + expect(wrapper.find(FormKeyUpdateWarning)).toHaveLength(1); + }); + + it('calls onApiTokenChange on form submit', () => { + const wrapper = shallow(); + + const preventDefault = jest.fn(); + wrapper.find(EuiForm).simulate('submit', { preventDefault }); + + expect(preventDefault).toHaveBeenCalled(); + expect(actions.onApiTokenChange).toHaveBeenCalled(); }); }); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/body.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/body.tsx index 2afba633ca8924..0395c77cf9d89f 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/body.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/body.tsx @@ -5,15 +5,41 @@ */ import React from 'react'; -import { EuiFlyoutBody } from '@elastic/eui'; +import { useValues, useActions } from 'kea'; +import { EuiFlyoutBody, EuiForm } from '@elastic/eui'; import { FlashMessages } from '../../../../shared/flash_messages'; +import { CredentialsLogic } from '../credentials_logic'; +import { ApiTokenTypes } from '../constants'; + +import { + FormKeyName, + FormKeyType, + FormKeyReadWriteAccess, + FormKeyEngineAccess, + FormKeyUpdateWarning, +} from './form_components'; export const CredentialsFlyoutBody: React.FC = () => { + const { onApiTokenChange } = useActions(CredentialsLogic); + const { activeApiToken, activeApiTokenExists } = useValues(CredentialsLogic); + return ( - Details go here + { + e.preventDefault(); + onApiTokenChange(); + }} + component="form" + > + + + {activeApiToken.type === ApiTokenTypes.Private && } + {activeApiToken.type !== ApiTokenTypes.Admin && } + + {activeApiTokenExists && } ); }; diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/footer.test.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/footer.test.tsx index 1ec3e4756c5c40..c31546472b036d 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/footer.test.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/footer.test.tsx @@ -18,6 +18,7 @@ describe('CredentialsFlyoutFooter', () => { }; const actions = { hideCredentialsForm: jest.fn(), + onApiTokenChange: jest.fn(), }; beforeEach(() => { @@ -59,6 +60,6 @@ describe('CredentialsFlyoutFooter', () => { const button = wrapper.find('[data-test-subj="APIKeyActionButton"]'); button.simulate('click'); - // TODO: Expect onApiTokenChange to have been called + expect(actions.onApiTokenChange).toHaveBeenCalled(); }); }); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/footer.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/footer.tsx index 7564560eade953..e59a75a578ba42 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/footer.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/footer.tsx @@ -18,7 +18,7 @@ import { i18n } from '@kbn/i18n'; import { CredentialsLogic } from '../credentials_logic'; export const CredentialsFlyoutFooter: React.FC = () => { - const { hideCredentialsForm } = useActions(CredentialsLogic); + const { hideCredentialsForm, onApiTokenChange } = useActions(CredentialsLogic); const { activeApiTokenExists } = useValues(CredentialsLogic); return ( @@ -33,7 +33,7 @@ export const CredentialsFlyoutFooter: React.FC = () => { window.alert('submit')} + onClick={onApiTokenChange} fill={true} color="secondary" iconType="check" diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/index.ts b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/index.ts new file mode 100644 index 00000000000000..ad39717ff8979f --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/index.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export { FormKeyName } from './key_name'; +export { FormKeyType } from './key_type'; +export { FormKeyReadWriteAccess } from './key_read_write_access'; +export { FormKeyEngineAccess } from './key_engine_access'; +export { FormKeyUpdateWarning } from './key_update_warning'; diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_engine_access.test.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_engine_access.test.tsx new file mode 100644 index 00000000000000..b4b092f17a6aa9 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_engine_access.test.tsx @@ -0,0 +1,135 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { setMockValues, setMockActions } from '../../../../../__mocks__/kea.mock'; + +import React from 'react'; +import { shallow } from 'enzyme'; +import { EuiRadio, EuiCheckbox } from '@elastic/eui'; + +import { FormKeyEngineAccess, EngineSelection } from './key_engine_access'; + +describe('FormKeyEngineAccess', () => { + const values = { + myRole: { canAccessAllEngines: true }, + fullEngineAccessChecked: true, + }; + const actions = { + setAccessAllEngines: jest.fn(), + }; + + beforeEach(() => { + jest.clearAllMocks(); + setMockValues(values); + setMockActions(actions); + }); + + it('renders', () => { + const wrapper = shallow(); + + expect(wrapper.find(EuiRadio)).toHaveLength(2); + expect(wrapper.find(EngineSelection)).toHaveLength(0); + }); + + it('hides the full access radio option if the user does not have access to all engines', () => { + setMockValues({ + ...values, + myRole: { canAccessAllEngines: false }, + }); + const wrapper = shallow(); + + expect(wrapper.find('#all_engines').prop('hidden')).toEqual(true); + }); + + it('controls the checked values for access radios', () => { + setMockValues({ + ...values, + fullEngineAccessChecked: true, + }); + const wrapper = shallow(); + + expect(wrapper.find('#all_engines').prop('checked')).toEqual(true); + expect(wrapper.find('#all_engines').prop('value')).toEqual('true'); + expect(wrapper.find('#specific_engines').prop('checked')).toEqual(false); + expect(wrapper.find('#specific_engines').prop('value')).toEqual('false'); + + setMockValues({ + ...values, + fullEngineAccessChecked: false, + }); + wrapper.setProps({}); // Re-render + + expect(wrapper.find('#all_engines').prop('checked')).toEqual(false); + expect(wrapper.find('#all_engines').prop('value')).toEqual('false'); + expect(wrapper.find('#specific_engines').prop('checked')).toEqual(true); + expect(wrapper.find('#specific_engines').prop('value')).toEqual('true'); + }); + + it('calls setAccessAllEngines when the radios are changed', () => { + const wrapper = shallow(); + + wrapper.find('#all_engines').simulate('change'); + expect(actions.setAccessAllEngines).toHaveBeenCalledWith(true); + + wrapper.find('#specific_engines').simulate('change'); + expect(actions.setAccessAllEngines).toHaveBeenCalledWith(false); + }); + + it('displays the engine selection panel if the limited access radio is selected', () => { + setMockValues({ + ...values, + fullEngineAccessChecked: false, + }); + const wrapper = shallow(); + + expect(wrapper.find(EngineSelection)).toHaveLength(1); + }); +}); + +describe('EngineSelection', () => { + const values = { + activeApiToken: { engines: [] }, + engines: [{ name: 'engine1' }, { name: 'engine2' }, { name: 'engine3' }], + }; + const actions = { + onEngineSelect: jest.fn(), + }; + + beforeEach(() => { + jest.clearAllMocks(); + setMockValues(values); + setMockActions(actions); + }); + + it('renders', () => { + const wrapper = shallow(); + + expect(wrapper.find('h4').text()).toEqual('Select Engines'); + expect(wrapper.find(EuiCheckbox)).toHaveLength(3); + expect(wrapper.find(EuiCheckbox).first().prop('label')).toEqual('engine1'); + }); + + it('controls the engines checked state', () => { + setMockValues({ + ...values, + activeApiToken: { engines: ['engine3'] }, + }); + const wrapper = shallow(); + + expect(wrapper.find(EuiCheckbox).first().prop('checked')).toEqual(false); + expect(wrapper.find(EuiCheckbox).last().prop('checked')).toEqual(true); + }); + + it('calls onEngineSelect when the checkboxes are changed', () => { + const wrapper = shallow(); + + wrapper.find(EuiCheckbox).first().simulate('change'); + expect(actions.onEngineSelect).toHaveBeenCalledWith('engine1'); + + wrapper.find(EuiCheckbox).last().simulate('change'); + expect(actions.onEngineSelect).toHaveBeenCalledWith('engine3'); + }); +}); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_engine_access.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_engine_access.tsx new file mode 100644 index 00000000000000..88e345d0f9966e --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_engine_access.tsx @@ -0,0 +1,133 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; +import { useValues, useActions } from 'kea'; +import { + EuiFormRow, + EuiRadio, + EuiCheckbox, + EuiText, + EuiTitle, + EuiSpacer, + EuiPanel, +} from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; + +import { AppLogic } from '../../../../app_logic'; +import { CredentialsLogic } from '../../credentials_logic'; + +export const FormKeyEngineAccess: React.FC = () => { + const { myRole } = useValues(AppLogic); + const { setAccessAllEngines } = useActions(CredentialsLogic); + const { fullEngineAccessChecked } = useValues(CredentialsLogic); + + return ( + <> + + + <> + + +

+ {i18n.translate( + 'xpack.enterpriseSearch.appSearch.credentials.formEngineAccess.fullAccess.label', + { defaultMessage: 'Full Engine Access' } + )} +

+ + + {i18n.translate( + 'xpack.enterpriseSearch.appSearch.credentials.formEngineAccess.fullAccess.helpText', + { defaultMessage: 'Access to all current and future Engines.' } + )} + + + } + hidden={!myRole.canAccessAllEngines} + checked={fullEngineAccessChecked} + value={fullEngineAccessChecked.toString()} + onChange={() => setAccessAllEngines(true)} + /> + + + +

+ {i18n.translate( + 'xpack.enterpriseSearch.appSearch.credentials.formEngineAccess.limitedAccess.label', + { defaultMessage: 'Limited Engine Access' } + )} +

+ + + {i18n.translate( + 'xpack.enterpriseSearch.appSearch.credentials.formEngineAccess.limitedAccess.helpText', + { defaultMessage: 'Limit key access to specific Engines.' } + )} + + + } + checked={!fullEngineAccessChecked} + value={(!fullEngineAccessChecked).toString()} + onChange={() => setAccessAllEngines(false)} + /> + + + {!fullEngineAccessChecked && } + + ); +}; + +export const EngineSelection: React.FC = () => { + const { onEngineSelect } = useActions(CredentialsLogic); + const { activeApiToken, engines } = useValues(CredentialsLogic); + + return ( + <> + + + +

+ {i18n.translate( + 'xpack.enterpriseSearch.appSearch.credentials.formEngineAccess.engineAccess.label', + { defaultMessage: 'Select Engines' } + )} +

+ + + {i18n.translate( + 'xpack.enterpriseSearch.appSearch.credentials.formEngineAccess.engineAccess.helpText', + { defaultMessage: 'Engines which the key can access:' } + )} + + + {engines.map((engine) => ( + onEngineSelect(engine.name)} + /> + ))} + + + ); +}; diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_name.test.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_name.test.tsx new file mode 100644 index 00000000000000..87f0f843dfa671 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_name.test.tsx @@ -0,0 +1,88 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { setMockValues, setMockActions } from '../../../../../__mocks__/kea.mock'; + +import React from 'react'; +import { shallow } from 'enzyme'; +import { EuiFieldText, EuiFormRow } from '@elastic/eui'; + +import { FormKeyName } from './'; + +describe('FormKeyName', () => { + const values = { + activeApiToken: { name: '' }, + activeApiTokenRawName: '', + activeApiTokenExists: false, + }; + const actions = { + setNameInputBlurred: jest.fn(), + setTokenName: jest.fn(), + }; + + beforeEach(() => { + jest.clearAllMocks(); + setMockValues(values); + setMockActions(actions); + }); + + it('renders', () => { + const wrapper = shallow(); + + expect(wrapper.find(EuiFieldText)).toHaveLength(1); + expect(wrapper.find(EuiFieldText).prop('placeholder')).toEqual('i.e., my-engine-key'); + expect(wrapper.find(EuiFieldText).prop('value')).toEqual(''); + expect(wrapper.find(EuiFieldText).prop('disabled')).toEqual(false); + expect(wrapper.find(EuiFormRow).prop('helpText')).toEqual(''); + }); + + it('shows help text if the raw name does not match the expected name', () => { + setMockValues({ + ...values, + activeApiToken: { name: 'my-engine-key' }, + activeApiTokenRawName: 'my engine key!!', + }); + const wrapper = shallow(); + + expect(wrapper.find(EuiFormRow).prop('helpText')).toEqual( + 'Your key will be named: my-engine-key' + ); + }); + + it('controls the input value', () => { + setMockValues({ + ...values, + activeApiTokenRawName: 'test', + }); + const wrapper = shallow(); + + expect(wrapper.find(EuiFieldText).prop('value')).toEqual('test'); + }); + + it('disables the input if editing an existing key', () => { + setMockValues({ + ...values, + activeApiTokenExists: true, + }); + const wrapper = shallow(); + + expect(wrapper.find(EuiFieldText).prop('disabled')).toEqual(true); + }); + + it('calls setTokenName when the input value is changed', () => { + const wrapper = shallow(); + wrapper.find(EuiFieldText).simulate('change', { target: { value: 'changed' } }); + + expect(actions.setTokenName).toHaveBeenCalledWith('changed'); + }); + + it('calls setNameInputBlurred when the user stops focusing the input', () => { + const wrapper = shallow(); + wrapper.find(EuiFieldText).simulate('blur'); + + expect(actions.setNameInputBlurred).toHaveBeenCalledWith(true); + }); +}); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_name.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_name.tsx new file mode 100644 index 00000000000000..fb8de2b244eccf --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_name.tsx @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; +import { useValues, useActions } from 'kea'; +import { EuiFormRow, EuiFieldText } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; + +import { CredentialsLogic } from '../../credentials_logic'; + +export const FormKeyName: React.FC = () => { + const { setNameInputBlurred, setTokenName } = useActions(CredentialsLogic); + const { + activeApiToken: { name }, + activeApiTokenRawName: rawName, + activeApiTokenExists, + } = useValues(CredentialsLogic); + + return ( + + setTokenName(e.target.value)} + onBlur={() => setNameInputBlurred(true)} + autoComplete="off" + maxLength={64} + disabled={activeApiTokenExists} + required + fullWidth + autoFocus + /> + + ); +}; diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_read_write_access.test.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_read_write_access.test.tsx new file mode 100644 index 00000000000000..2f1be1b07cbe19 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_read_write_access.test.tsx @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { setMockValues, setMockActions } from '../../../../../__mocks__/kea.mock'; + +import React from 'react'; +import { shallow } from 'enzyme'; +import { EuiCheckbox } from '@elastic/eui'; + +import { FormKeyReadWriteAccess } from './'; + +describe('FormKeyReadWriteAccess', () => { + const values = { + activeApiToken: { read: false, write: false }, + }; + const actions = { + setTokenReadWrite: jest.fn(), + }; + + beforeEach(() => { + jest.clearAllMocks(); + setMockValues(values); + setMockActions(actions); + }); + + it('renders', () => { + const wrapper = shallow(); + + expect(wrapper.find('h3').text()).toEqual('Read and Write Access Levels'); + expect(wrapper.find(EuiCheckbox)).toHaveLength(2); + }); + + it('controls the checked state for the read checkbox', () => { + setMockValues({ + ...values, + activeApiToken: { read: true, write: false }, + }); + const wrapper = shallow(); + + expect(wrapper.find('#read').prop('checked')).toEqual(true); + expect(wrapper.find('#write').prop('checked')).toEqual(false); + }); + + it('controls the checked state for the write checkbox', () => { + setMockValues({ + ...values, + activeApiToken: { read: false, write: true }, + }); + const wrapper = shallow(); + + expect(wrapper.find('#read').prop('checked')).toEqual(false); + expect(wrapper.find('#write').prop('checked')).toEqual(true); + }); + + it('calls setTokenReadWrite when the checkboxes are changed', () => { + const wrapper = shallow(); + + wrapper.find('#read').simulate('change', { target: { name: 'read', checked: true } }); + expect(actions.setTokenReadWrite).toHaveBeenCalledWith({ name: 'read', checked: true }); + + wrapper.find('#write').simulate('change', { target: { name: 'write', checked: false } }); + expect(actions.setTokenReadWrite).toHaveBeenCalledWith({ name: 'write', checked: false }); + }); +}); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_read_write_access.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_read_write_access.tsx new file mode 100644 index 00000000000000..a02b00b6ad3770 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_read_write_access.tsx @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; +import { useValues, useActions } from 'kea'; +import { EuiCheckbox, EuiText, EuiTitle, EuiSpacer, EuiPanel } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; + +import { CredentialsLogic } from '../../credentials_logic'; +import { ITokenReadWrite } from '../../types'; + +export const FormKeyReadWriteAccess: React.FC = () => { + const { setTokenReadWrite } = useActions(CredentialsLogic); + const { activeApiToken } = useValues(CredentialsLogic); + + return ( + <> + + + +

+ {i18n.translate('xpack.enterpriseSearch.appSearch.credentials.formReadWrite.label', { + defaultMessage: 'Read and Write Access Levels', + })} +

+ + + {i18n.translate('xpack.enterpriseSearch.appSearch.credentials.formReadWrite.helpText', { + defaultMessage: 'Only applies to Private API Keys.', + })} + + + setTokenReadWrite(e.target as ITokenReadWrite)} + label={i18n.translate( + 'xpack.enterpriseSearch.appSearch.credentials.formReadWrite.readLabel', + { defaultMessage: 'Read Access' } + )} + /> + setTokenReadWrite(e.target as ITokenReadWrite)} + label={i18n.translate( + 'xpack.enterpriseSearch.appSearch.credentials.formReadWrite.writeLabel', + { defaultMessage: 'Write Access' } + )} + /> + + + ); +}; diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_type.test.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_type.test.tsx new file mode 100644 index 00000000000000..d07a705b2d90bf --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_type.test.tsx @@ -0,0 +1,80 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { setMockValues, setMockActions } from '../../../../../__mocks__/kea.mock'; + +import React from 'react'; +import { shallow } from 'enzyme'; +import { EuiSelect } from '@elastic/eui'; + +import { ApiTokenTypes, TOKEN_TYPE_INFO } from '../../constants'; +import { FormKeyType } from './'; + +describe('FormKeyType', () => { + const values = { + myRole: { credentialTypes: ['search', 'private', 'admin'] }, + activeApiToken: { type: ApiTokenTypes.Private }, + activeApiTokenExists: false, + }; + const actions = { + setTokenType: jest.fn(), + }; + + beforeEach(() => { + jest.clearAllMocks(); + setMockValues(values); + setMockActions(actions); + }); + + it('renders', () => { + const wrapper = shallow(); + + expect(wrapper.find(EuiSelect)).toHaveLength(1); + expect(wrapper.find(EuiSelect).prop('placeholder')).toEqual('Select a key type'); + expect(wrapper.find(EuiSelect).prop('options')).toEqual(TOKEN_TYPE_INFO); + expect(wrapper.find(EuiSelect).prop('value')).toEqual(ApiTokenTypes.Private); + expect(wrapper.find(EuiSelect).prop('disabled')).toEqual(false); + }); + + it('only shows the type options that the user has access to', () => { + setMockValues({ + ...values, + myRole: { credentialTypes: ['search'] }, + }); + const wrapper = shallow(); + + expect(wrapper.find(EuiSelect).prop('options')).toEqual([ + expect.objectContaining({ value: ApiTokenTypes.Search }), + ]); + }); + + it('controls the select value', () => { + setMockValues({ + ...values, + activeApiToken: { type: ApiTokenTypes.Search }, + }); + const wrapper = shallow(); + + expect(wrapper.find(EuiSelect).prop('value')).toEqual(ApiTokenTypes.Search); + }); + + it('disables the select if editing an existing key', () => { + setMockValues({ + ...values, + activeApiTokenExists: true, + }); + const wrapper = shallow(); + + expect(wrapper.find(EuiSelect).prop('disabled')).toEqual(true); + }); + + it('calls setTokenType when the select value is changed', () => { + const wrapper = shallow(); + wrapper.find(EuiSelect).simulate('change', { target: { value: ApiTokenTypes.Admin } }); + + expect(actions.setTokenType).toHaveBeenCalledWith(ApiTokenTypes.Admin); + }); +}); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_type.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_type.tsx new file mode 100644 index 00000000000000..7268c12614e8b5 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_type.tsx @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; +import { useValues, useActions } from 'kea'; +import { EuiFormRow, EuiSelect, EuiText, EuiLink } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; + +import { AppLogic } from '../../../../app_logic'; +import { CredentialsLogic } from '../../credentials_logic'; +import { TOKEN_TYPE_DESCRIPTION, TOKEN_TYPE_INFO, DOCS_HREF } from '../../constants'; + +export const FormKeyType: React.FC = () => { + const { myRole } = useValues(AppLogic); + const { setTokenType } = useActions(CredentialsLogic); + const { activeApiToken, activeApiTokenExists } = useValues(CredentialsLogic); + + const tokenDescription = TOKEN_TYPE_DESCRIPTION[activeApiToken.type]; + const tokenOptions = TOKEN_TYPE_INFO.filter((typeInfo) => + myRole?.credentialTypes?.includes(typeInfo.value) + ); + + return ( + +

+ {tokenDescription}{' '} + + {i18n.translate('xpack.enterpriseSearch.appSearch.credentials.documentationLink1', { + defaultMessage: 'Visit the documentation', + })} + {' '} + {i18n.translate('xpack.enterpriseSearch.appSearch.credentials.documentationLink2', { + defaultMessage: 'to learn more about keys.', + })} +

+ + } + > + setTokenType(e.target.value)} + placeholder={i18n.translate( + 'xpack.enterpriseSearch.appSearch.credentials.formType.placeholder', + { defaultMessage: 'Select a key type' } + )} + disabled={activeApiTokenExists} + required + fullWidth + /> + + ); +}; diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_update_warning.test.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_update_warning.test.tsx new file mode 100644 index 00000000000000..c0ff892c220c76 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_update_warning.test.tsx @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; +import { shallow } from 'enzyme'; +import { EuiCallOut } from '@elastic/eui'; + +import { FormKeyUpdateWarning } from './'; + +describe('FormKeyUpdateWarning', () => { + it('renders', () => { + const wrapper = shallow(); + expect(wrapper.find(EuiCallOut)).toHaveLength(1); + }); +}); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_update_warning.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_update_warning.tsx new file mode 100644 index 00000000000000..7e7aaa583325d5 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_flyout/form_components/key_update_warning.tsx @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; +import { EuiSpacer, EuiCallOut } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; + +export const FormKeyUpdateWarning: React.FC = () => ( + <> + + +

+ {i18n.translate('xpack.enterpriseSearch.appSearch.credentials.updateWarning', { + defaultMessage: + 'Existing API keys may be shared between users. Changing permissions for this key will affect all users who have access to this key.', + })} +

+ + +); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_logic.test.ts b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_logic.test.ts index 8eb0f7582516de..de79862b540ba8 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_logic.test.ts +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_logic.test.ts @@ -6,13 +6,12 @@ import { resetContext } from 'kea'; -import { CredentialsLogic } from './credentials_logic'; -import { ApiTokenTypes } from './constants'; - +import { mockHttpValues } from '../../../__mocks__'; jest.mock('../../../shared/http', () => ({ - HttpLogic: { values: { http: { get: jest.fn(), delete: jest.fn() } } }, + HttpLogic: { values: mockHttpValues }, })); -import { HttpLogic } from '../../../shared/http'; +const { http } = mockHttpValues; + jest.mock('../../../shared/flash_messages', () => ({ FlashMessagesLogic: { actions: { clearFlashMessages: jest.fn() } }, setSuccessMessage: jest.fn(), @@ -24,6 +23,17 @@ import { flashAPIErrors, } from '../../../shared/flash_messages'; +jest.mock('../../app_logic', () => ({ + AppLogic: { + selectors: { myRole: jest.fn(() => ({})) }, + values: { myRole: jest.fn(() => ({})) }, + }, +})); +import { AppLogic } from '../../app_logic'; + +import { ApiTokenTypes } from './constants'; +import { CredentialsLogic } from './credentials_logic'; + describe('CredentialsLogic', () => { const DEFAULT_VALUES = { activeApiToken: { @@ -44,6 +54,7 @@ describe('CredentialsLogic', () => { meta: {}, nameInputBlurred: false, shouldShowCredentialsForm: false, + fullEngineAccessChecked: false, }; const mount = (defaults?: object) => { @@ -1081,10 +1092,10 @@ describe('CredentialsLogic', () => { mount(); jest.spyOn(CredentialsLogic.actions, 'setCredentialsData').mockImplementationOnce(() => {}); const promise = Promise.resolve({ meta, results }); - (HttpLogic.values.http.get as jest.Mock).mockReturnValue(promise); + http.get.mockReturnValue(promise); CredentialsLogic.actions.fetchCredentials(2); - expect(HttpLogic.values.http.get).toHaveBeenCalledWith('/api/app_search/credentials', { + expect(http.get).toHaveBeenCalledWith('/api/app_search/credentials', { query: { 'page[current]': 2, }, @@ -1096,7 +1107,7 @@ describe('CredentialsLogic', () => { it('handles errors', async () => { mount(); const promise = Promise.reject('An error occured'); - (HttpLogic.values.http.get as jest.Mock).mockReturnValue(promise); + http.get.mockReturnValue(promise); CredentialsLogic.actions.fetchCredentials(); try { @@ -1114,12 +1125,10 @@ describe('CredentialsLogic', () => { .spyOn(CredentialsLogic.actions, 'setCredentialsDetails') .mockImplementationOnce(() => {}); const promise = Promise.resolve(credentialsDetails); - (HttpLogic.values.http.get as jest.Mock).mockReturnValue(promise); + http.get.mockReturnValue(promise); CredentialsLogic.actions.fetchDetails(); - expect(HttpLogic.values.http.get).toHaveBeenCalledWith( - '/api/app_search/credentials/details' - ); + expect(http.get).toHaveBeenCalledWith('/api/app_search/credentials/details'); await promise; expect(CredentialsLogic.actions.setCredentialsDetails).toHaveBeenCalledWith( credentialsDetails @@ -1129,7 +1138,7 @@ describe('CredentialsLogic', () => { it('handles errors', async () => { mount(); const promise = Promise.reject('An error occured'); - (HttpLogic.values.http.get as jest.Mock).mockReturnValue(promise); + http.get.mockReturnValue(promise); CredentialsLogic.actions.fetchDetails(); try { @@ -1147,12 +1156,10 @@ describe('CredentialsLogic', () => { mount(); jest.spyOn(CredentialsLogic.actions, 'onApiKeyDelete').mockImplementationOnce(() => {}); const promise = Promise.resolve(); - (HttpLogic.values.http.delete as jest.Mock).mockReturnValue(promise); + http.delete.mockReturnValue(promise); CredentialsLogic.actions.deleteApiKey(tokenName); - expect(HttpLogic.values.http.delete).toHaveBeenCalledWith( - `/api/app_search/credentials/${tokenName}` - ); + expect(http.delete).toHaveBeenCalledWith(`/api/app_search/credentials/${tokenName}`); await promise; expect(CredentialsLogic.actions.onApiKeyDelete).toHaveBeenCalledWith(tokenName); expect(setSuccessMessage).toHaveBeenCalled(); @@ -1161,7 +1168,7 @@ describe('CredentialsLogic', () => { it('handles errors', async () => { mount(); const promise = Promise.reject('An error occured'); - (HttpLogic.values.http.delete as jest.Mock).mockReturnValue(promise); + http.delete.mockReturnValue(promise); CredentialsLogic.actions.deleteApiKey(tokenName); try { @@ -1171,9 +1178,189 @@ describe('CredentialsLogic', () => { } }); }); + + describe('onApiTokenChange', () => { + it('calls a POST API endpoint that creates a new token if the active token does not exist yet', async () => { + const createdToken = { + name: 'new-key', + type: ApiTokenTypes.Admin, + }; + mount({ + activeApiToken: createdToken, + }); + jest.spyOn(CredentialsLogic.actions, 'onApiTokenCreateSuccess'); + const promise = Promise.resolve(createdToken); + http.post.mockReturnValue(promise); + + CredentialsLogic.actions.onApiTokenChange(); + expect(http.post).toHaveBeenCalledWith('/api/app_search/credentials', { + body: JSON.stringify(createdToken), + }); + await promise; + expect(CredentialsLogic.actions.onApiTokenCreateSuccess).toHaveBeenCalledWith(createdToken); + expect(setSuccessMessage).toHaveBeenCalled(); + }); + + it('calls a PUT endpoint that updates the active token if it already exists', async () => { + const updatedToken = { + name: 'test-key', + type: ApiTokenTypes.Private, + read: true, + write: false, + access_all_engines: false, + engines: ['engine1'], + }; + mount({ + activeApiToken: { + ...updatedToken, + id: 'some-id', + }, + }); + jest.spyOn(CredentialsLogic.actions, 'onApiTokenUpdateSuccess'); + const promise = Promise.resolve(updatedToken); + http.put.mockReturnValue(promise); + + CredentialsLogic.actions.onApiTokenChange(); + expect(http.put).toHaveBeenCalledWith('/api/app_search/credentials/test-key', { + body: JSON.stringify(updatedToken), + }); + await promise; + expect(CredentialsLogic.actions.onApiTokenUpdateSuccess).toHaveBeenCalledWith(updatedToken); + expect(setSuccessMessage).toHaveBeenCalled(); + }); + + it('handles errors', async () => { + mount(); + const promise = Promise.reject('An error occured'); + http.post.mockReturnValue(promise); + + CredentialsLogic.actions.onApiTokenChange(); + try { + await promise; + } catch { + expect(flashAPIErrors).toHaveBeenCalledWith('An error occured'); + } + }); + + describe('token type data', () => { + it('does not send extra read/write/engine access data for admin tokens', () => { + const correctAdminToken = { + name: 'bogus-admin', + type: ApiTokenTypes.Admin, + }; + const extraData = { + read: true, + write: true, + access_all_engines: true, + }; + mount({ activeApiToken: { ...correctAdminToken, ...extraData } }); + + CredentialsLogic.actions.onApiTokenChange(); + expect(http.post).toHaveBeenCalledWith('/api/app_search/credentials', { + body: JSON.stringify(correctAdminToken), + }); + }); + + it('does not send extra read/write access data for search tokens', () => { + const correctSearchToken = { + name: 'bogus-search', + type: ApiTokenTypes.Search, + access_all_engines: false, + engines: ['some-engine'], + }; + const extraData = { + read: true, + write: false, + }; + mount({ activeApiToken: { ...correctSearchToken, ...extraData } }); + + CredentialsLogic.actions.onApiTokenChange(); + expect(http.post).toHaveBeenCalledWith('/api/app_search/credentials', { + body: JSON.stringify(correctSearchToken), + }); + }); + + // Private tokens send all data per the PUT test above. + // If that ever changes, we should capture that in another test here. + }); + }); + + describe('onEngineSelect', () => { + it('calls addEngineName if the engine is not selected', () => { + mount({ + activeApiToken: { + ...DEFAULT_VALUES.activeApiToken, + engines: [], + }, + }); + jest.spyOn(CredentialsLogic.actions, 'addEngineName'); + + CredentialsLogic.actions.onEngineSelect('engine1'); + expect(CredentialsLogic.actions.addEngineName).toHaveBeenCalledWith('engine1'); + expect(CredentialsLogic.values.activeApiToken.engines).toEqual(['engine1']); + }); + + it('calls removeEngineName if the engine is already selected', () => { + mount({ + activeApiToken: { + ...DEFAULT_VALUES.activeApiToken, + engines: ['engine1', 'engine2'], + }, + }); + jest.spyOn(CredentialsLogic.actions, 'removeEngineName'); + + CredentialsLogic.actions.onEngineSelect('engine1'); + expect(CredentialsLogic.actions.removeEngineName).toHaveBeenCalledWith('engine1'); + expect(CredentialsLogic.values.activeApiToken.engines).toEqual(['engine2']); + }); + }); }); describe('selectors', () => { + describe('fullEngineAccessChecked', () => { + it('should be true if active token is set to access all engines and the user can access all engines', () => { + (AppLogic.selectors.myRole as jest.Mock).mockReturnValueOnce({ + canAccessAllEngines: true, + }); + mount({ + activeApiToken: { + ...DEFAULT_VALUES.activeApiToken, + access_all_engines: true, + }, + }); + + expect(CredentialsLogic.values.fullEngineAccessChecked).toEqual(true); + }); + + it('should be false if the token is not set to access all engines', () => { + (AppLogic.selectors.myRole as jest.Mock).mockReturnValueOnce({ + canAccessAllEngines: true, + }); + mount({ + activeApiToken: { + ...DEFAULT_VALUES.activeApiToken, + access_all_engines: false, + }, + }); + + expect(CredentialsLogic.values.fullEngineAccessChecked).toEqual(false); + }); + + it('should be false if the user cannot acess all engines', () => { + (AppLogic.selectors.myRole as jest.Mock).mockReturnValueOnce({ + canAccessAllEngines: false, + }); + mount({ + activeApiToken: { + ...DEFAULT_VALUES.activeApiToken, + access_all_engines: true, + }, + }); + + expect(CredentialsLogic.values.fullEngineAccessChecked).toEqual(false); + }); + }); + describe('activeApiTokenExists', () => { it('should be false if the token has no id', () => { mount({ diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_logic.ts b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_logic.ts index 40966d64212f6f..7b8b864b3a317c 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_logic.ts +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/credentials/credentials_logic.ts @@ -7,7 +7,7 @@ import { kea, MakeLogicType } from 'kea'; import { formatApiName } from '../../utils/format_api_name'; -import { ApiTokenTypes, DELETE_MESSAGE } from './constants'; +import { ApiTokenTypes, CREATE_MESSAGE, UPDATE_MESSAGE, DELETE_MESSAGE } from './constants'; import { HttpLogic } from '../../../shared/http'; import { @@ -15,12 +15,13 @@ import { setSuccessMessage, flashAPIErrors, } from '../../../shared/flash_messages'; +import { AppLogic } from '../../app_logic'; import { IMeta } from '../../../../../common/types'; import { IEngine } from '../../types'; import { IApiToken, ICredentialsDetails, ITokenReadWrite } from './types'; -const defaultApiToken: IApiToken = { +export const defaultApiToken: IApiToken = { name: '', type: ApiTokenTypes.Private, read: true, @@ -49,6 +50,8 @@ interface ICredentialsLogicActions { fetchCredentials(page?: number): number; fetchDetails(): { value: boolean }; deleteApiKey(tokenName: string): string; + onApiTokenChange(): void; + onEngineSelect(engineName: string): string; } interface ICredentialsLogicValues { @@ -92,6 +95,8 @@ export const CredentialsLogic = kea< fetchCredentials: (page) => page, fetchDetails: true, deleteApiKey: (tokenName) => tokenName, + onApiTokenChange: () => null, + onEngineSelect: (engineName) => engineName, }), reducers: () => ({ apiTokens: [ @@ -204,7 +209,11 @@ export const CredentialsLogic = kea< ], }), selectors: ({ selectors }) => ({ - // TODO fullEngineAccessChecked from ent-search + fullEngineAccessChecked: [ + () => [AppLogic.selectors.myRole, selectors.activeApiToken], + (myRole, activeApiToken) => + !!(myRole.canAccessAllEngines && activeApiToken.access_all_engines), + ], dataLoading: [ () => [selectors.isCredentialsDetailsComplete, selectors.isCredentialsDataComplete], (isCredentialsDetailsComplete, isCredentialsDataComplete) => { @@ -255,7 +264,45 @@ export const CredentialsLogic = kea< flashAPIErrors(e); } }, - // TODO onApiTokenChange from ent-search - // TODO onEngineSelect from ent-search + onApiTokenChange: async () => { + const { id, name, engines, type, read, write } = values.activeApiToken; + + const data: IApiToken = { + name, + type, + }; + if (type === ApiTokenTypes.Private) { + data.read = read; + data.write = write; + } + if (type !== ApiTokenTypes.Admin) { + data.access_all_engines = values.fullEngineAccessChecked; + data.engines = engines; + } + + try { + const { http } = HttpLogic.values; + const body = JSON.stringify(data); + + if (id) { + const response = await http.put(`/api/app_search/credentials/${name}`, { body }); + actions.onApiTokenUpdateSuccess(response); + setSuccessMessage(UPDATE_MESSAGE); + } else { + const response = await http.post('/api/app_search/credentials', { body }); + actions.onApiTokenCreateSuccess(response); + setSuccessMessage(CREATE_MESSAGE); + } + } catch (e) { + flashAPIErrors(e); + } + }, + onEngineSelect: (engineName: string) => { + if (values.activeApiToken?.engines?.includes(engineName)) { + actions.removeEngineName(engineName); + } else { + actions.addEngineName(engineName); + } + }, }), }); diff --git a/x-pack/plugins/enterprise_search/server/routes/app_search/credentials.test.ts b/x-pack/plugins/enterprise_search/server/routes/app_search/credentials.test.ts index 6b5f4a05b3aa67..357b49de934122 100644 --- a/x-pack/plugins/enterprise_search/server/routes/app_search/credentials.test.ts +++ b/x-pack/plugins/enterprise_search/server/routes/app_search/credentials.test.ts @@ -41,6 +41,115 @@ describe('credentials routes', () => { }); }); + describe('POST /api/app_search/credentials', () => { + let mockRouter: MockRouter; + + beforeEach(() => { + jest.clearAllMocks(); + mockRouter = new MockRouter({ method: 'post', payload: 'body' }); + + registerCredentialsRoutes({ + ...mockDependencies, + router: mockRouter.router, + }); + }); + + it('creates a request handler', () => { + expect(mockRequestHandler.createRequest).toHaveBeenCalledWith({ + path: '/as/credentials/collection', + }); + }); + + describe('validates', () => { + describe('admin keys', () => { + it('correctly', () => { + const request = { + body: { + name: 'admin-key', + type: 'admin', + }, + }; + mockRouter.shouldValidate(request); + }); + + it('throws on unnecessary properties', () => { + const request = { + body: { + name: 'admin-key', + type: 'admin', + read: true, + access_all_engines: true, + }, + }; + mockRouter.shouldThrow(request); + }); + }); + + describe('private keys', () => { + it('correctly', () => { + const request = { + body: { + name: 'private-key', + type: 'private', + read: true, + write: false, + access_all_engines: false, + engines: ['engine1', 'engine2'], + }, + }; + mockRouter.shouldValidate(request); + }); + + it('throws on missing keys', () => { + const request = { + body: { + name: 'private-key', + type: 'private', + }, + }; + mockRouter.shouldThrow(request); + }); + }); + + describe('search keys', () => { + it('correctly', () => { + const request = { + body: { + name: 'search-key', + type: 'search', + access_all_engines: true, + }, + }; + mockRouter.shouldValidate(request); + }); + + it('throws on missing keys', () => { + const request = { + body: { + name: 'search-key', + type: 'search', + }, + }; + mockRouter.shouldThrow(request); + }); + + it('throws on extra keys', () => { + const request = { + body: { + name: 'search-key', + type: 'search', + read: true, + write: false, + access_all_engines: false, + engines: ['engine1', 'engine2'], + }, + }; + mockRouter.shouldThrow(request); + }); + }); + }); + }); + describe('GET /api/app_search/credentials/details', () => { let mockRouter: MockRouter; @@ -61,6 +170,123 @@ describe('credentials routes', () => { }); }); + describe('PUT /api/app_search/credentials/{name}', () => { + let mockRouter: MockRouter; + + beforeEach(() => { + jest.clearAllMocks(); + mockRouter = new MockRouter({ method: 'put', payload: 'body' }); + + registerCredentialsRoutes({ + ...mockDependencies, + router: mockRouter.router, + }); + }); + + it('creates a request to enterprise search', () => { + const mockRequest = { + params: { + name: 'abc123', + }, + }; + + mockRouter.callRoute(mockRequest); + + expect(mockRequestHandler.createRequest).toHaveBeenCalledWith({ + path: '/as/credentials/abc123', + }); + }); + + describe('validates', () => { + describe('admin keys', () => { + it('correctly', () => { + const request = { + body: { + name: 'admin-key', + type: 'admin', + }, + }; + mockRouter.shouldValidate(request); + }); + + it('throws on unnecessary properties', () => { + const request = { + body: { + name: 'admin-key', + type: 'admin', + read: true, + access_all_engines: true, + }, + }; + mockRouter.shouldThrow(request); + }); + }); + + describe('private keys', () => { + it('correctly', () => { + const request = { + body: { + name: 'private-key', + type: 'private', + read: true, + write: false, + access_all_engines: false, + engines: ['engine1', 'engine2'], + }, + }; + mockRouter.shouldValidate(request); + }); + + it('throws on missing keys', () => { + const request = { + body: { + name: 'private-key', + type: 'private', + }, + }; + mockRouter.shouldThrow(request); + }); + }); + + describe('search keys', () => { + it('correctly', () => { + const request = { + body: { + name: 'search-key', + type: 'search', + access_all_engines: true, + }, + }; + mockRouter.shouldValidate(request); + }); + + it('throws on missing keys', () => { + const request = { + body: { + name: 'search-key', + type: 'search', + }, + }; + mockRouter.shouldThrow(request); + }); + + it('throws on extra keys', () => { + const request = { + body: { + name: 'search-key', + type: 'search', + read: true, + write: false, + access_all_engines: false, + engines: ['engine1', 'engine2'], + }, + }; + mockRouter.shouldThrow(request); + }); + }); + }); + }); + describe('DELETE /api/app_search/credentials/{name}', () => { let mockRouter: MockRouter; diff --git a/x-pack/plugins/enterprise_search/server/routes/app_search/credentials.ts b/x-pack/plugins/enterprise_search/server/routes/app_search/credentials.ts index 0f2c1133192c57..85d213c82dd055 100644 --- a/x-pack/plugins/enterprise_search/server/routes/app_search/credentials.ts +++ b/x-pack/plugins/enterprise_search/server/routes/app_search/credentials.ts @@ -8,10 +8,32 @@ import { schema } from '@kbn/config-schema'; import { IRouteDependencies } from '../../plugin'; +const tokenSchema = schema.oneOf([ + schema.object({ + name: schema.string(), + type: schema.literal('admin'), + }), + schema.object({ + name: schema.string(), + type: schema.literal('private'), + read: schema.boolean(), + write: schema.boolean(), + access_all_engines: schema.boolean(), + engines: schema.maybe(schema.arrayOf(schema.string())), + }), + schema.object({ + name: schema.string(), + type: schema.literal('search'), + access_all_engines: schema.boolean(), + engines: schema.maybe(schema.arrayOf(schema.string())), + }), +]); + export function registerCredentialsRoutes({ router, enterpriseSearchRequestHandler, }: IRouteDependencies) { + // Credentials API router.get( { path: '/api/app_search/credentials', @@ -25,6 +47,19 @@ export function registerCredentialsRoutes({ path: '/as/credentials/collection', }) ); + router.post( + { + path: '/api/app_search/credentials', + validate: { + body: tokenSchema, + }, + }, + enterpriseSearchRequestHandler.createRequest({ + path: '/as/credentials/collection', + }) + ); + + // TODO: It would be great to remove this someday router.get( { path: '/api/app_search/credentials/details', @@ -34,6 +69,24 @@ export function registerCredentialsRoutes({ path: '/as/credentials/details', }) ); + + // Single credential API + router.put( + { + path: '/api/app_search/credentials/{name}', + validate: { + params: schema.object({ + name: schema.string(), + }), + body: tokenSchema, + }, + }, + async (context, request, response) => { + return enterpriseSearchRequestHandler.createRequest({ + path: `/as/credentials/${request.params.name}`, + })(context, request, response); + } + ); router.delete( { path: '/api/app_search/credentials/{name}', diff --git a/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/edit_policy/constants.ts b/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/edit_policy/constants.ts index bd845b0a7d9a76..0a96146339a58b 100644 --- a/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/edit_policy/constants.ts +++ b/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/edit_policy/constants.ts @@ -14,17 +14,52 @@ export const DEFAULT_POLICY: PolicyFromES = { version: 1, modified_date: Date.now().toString(), policy: { - name: '', + name: 'my_policy', + phases: { + hot: { + min_age: '0ms', + actions: { + rollover: { + max_age: '30d', + max_size: '50gb', + }, + }, + }, + }, + }, + name: 'my_policy', +}; + +export const POLICY_WITH_INCLUDE_EXCLUDE: PolicyFromES = { + version: 1, + modified_date: Date.now().toString(), + policy: { + name: 'my_policy', phases: { hot: { min_age: '123ms', actions: { - rollover: {}, + rollover: { + max_age: '30d', + max_size: '50gb', + }, + }, + }, + warm: { + actions: { + allocate: { + include: { + abc: '123', + }, + exclude: { + def: '456', + }, + }, }, }, }, }, - name: '', + name: 'my_policy', }; export const DELETE_PHASE_POLICY: PolicyFromES = { @@ -60,3 +95,58 @@ export const DELETE_PHASE_POLICY: PolicyFromES = { }, name: POLICY_NAME, }; + +export const POLICY_WITH_NODE_ATTR_AND_OFF_ALLOCATION: PolicyFromES = { + version: 1, + modified_date: Date.now().toString(), + policy: { + phases: { + hot: { + min_age: '0ms', + actions: { + rollover: { + max_size: '50gb', + }, + }, + }, + warm: { + actions: { + allocate: { + require: {}, + include: { test: '123' }, + exclude: {}, + }, + }, + }, + cold: { + actions: { + migrate: { enabled: false }, + }, + }, + }, + name: POLICY_NAME, + }, + name: POLICY_NAME, +}; + +export const POLICY_WITH_NODE_ROLE_ALLOCATION: PolicyFromES = { + version: 1, + modified_date: Date.now().toString(), + policy: { + phases: { + hot: { + min_age: '0ms', + actions: { + rollover: { + max_size: '50gb', + }, + }, + }, + warm: { + actions: {}, + }, + }, + name: POLICY_NAME, + }, + name: POLICY_NAME, +}; diff --git a/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/edit_policy/edit_policy.helpers.tsx b/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/edit_policy/edit_policy.helpers.tsx index 0cfccba7613094..1716f124b0c83a 100644 --- a/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/edit_policy/edit_policy.helpers.tsx +++ b/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/edit_policy/edit_policy.helpers.tsx @@ -9,11 +9,16 @@ import { act } from 'react-dom/test-utils'; import { registerTestBed, TestBedConfig } from '../../../../../test_utils'; +import { EditPolicy } from '../../../public/application/sections/edit_policy'; +import { DataTierAllocationType } from '../../../public/application/sections/edit_policy/types'; + +import { Phases as PolicyPhases } from '../../../common/types'; + +type Phases = keyof PolicyPhases; + import { POLICY_NAME } from './constants'; import { TestSubjects } from '../helpers'; -import { EditPolicy } from '../../../public/application/sections/edit_policy'; - jest.mock('@elastic/eui', () => { const original = jest.requireActual('@elastic/eui'); @@ -52,7 +57,23 @@ export type EditPolicyTestBed = SetupReturn extends Promise ? U : Setup export const setup = async () => { const testBed = await initTestBed(); - const { find, component } = testBed; + const { find, component, form } = testBed; + + const createFormToggleAction = (dataTestSubject: string) => async (checked: boolean) => { + await act(async () => { + form.toggleEuiSwitch(dataTestSubject, checked); + }); + component.update(); + }; + + function createFormSetValueAction(dataTestSubject: string) { + return async (value: V) => { + await act(async () => { + form.setInputValue(dataTestSubject, value); + }); + component.update(); + }; + } const setWaitForSnapshotPolicy = async (snapshotPolicyName: string) => { act(() => { @@ -68,12 +89,7 @@ export const setup = async () => { component.update(); }; - const toggleRollover = async (checked: boolean) => { - await act(async () => { - find('rolloverSwitch').simulate('click', { target: { checked } }); - }); - component.update(); - }; + const toggleRollover = createFormToggleAction('rolloverSwitch'); const setMaxSize = async (value: string, units?: string) => { await act(async () => { @@ -87,12 +103,7 @@ export const setup = async () => { component.update(); }; - const setMaxDocs = async (value: string) => { - await act(async () => { - find('hot-selectedMaxDocuments').simulate('change', { target: { value } }); - }); - component.update(); - }; + const setMaxDocs = createFormSetValueAction('hot-selectedMaxDocuments'); const setMaxAge = async (value: string, units?: string) => { await act(async () => { @@ -104,32 +115,56 @@ export const setup = async () => { component.update(); }; - const toggleForceMerge = (phase: string) => async (checked: boolean) => { - await act(async () => { - find(`${phase}-forceMergeSwitch`).simulate('click', { target: { checked } }); + const toggleForceMerge = (phase: Phases) => createFormToggleAction(`${phase}-forceMergeSwitch`); + + const setForcemergeSegmentsCount = (phase: Phases) => + createFormSetValueAction(`${phase}-selectedForceMergeSegments`); + + const setBestCompression = (phase: Phases) => createFormToggleAction(`${phase}-bestCompression`); + + const setIndexPriority = (phase: Phases) => + createFormSetValueAction(`${phase}-phaseIndexPriority`); + + const enable = (phase: Phases) => createFormToggleAction(`enablePhaseSwitch-${phase}`); + + const warmPhaseOnRollover = createFormToggleAction(`warm-warmPhaseOnRollover`); + + const setMinAgeValue = (phase: Phases) => createFormSetValueAction(`${phase}-selectedMinimumAge`); + + const setMinAgeUnits = (phase: Phases) => + createFormSetValueAction(`${phase}-selectedMinimumAgeUnits`); + + const setDataAllocation = (phase: Phases) => async (value: DataTierAllocationType) => { + act(() => { + find(`${phase}-dataTierAllocationControls.dataTierSelect`).simulate('click'); }); component.update(); - }; - - const setForcemergeSegmentsCount = (phase: string) => async (value: string) => { await act(async () => { - find(`${phase}-selectedForceMergeSegments`).simulate('change', { target: { value } }); + switch (value) { + case 'node_roles': + find(`${phase}-dataTierAllocationControls.defaultDataAllocationOption`).simulate('click'); + break; + case 'node_attrs': + find(`${phase}-dataTierAllocationControls.customDataAllocationOption`).simulate('click'); + break; + default: + find(`${phase}-dataTierAllocationControls.noneDataAllocationOption`).simulate('click'); + } }); component.update(); }; - const setBestCompression = (phase: string) => async (checked: boolean) => { - await act(async () => { - find(`${phase}-bestCompression`).simulate('click', { target: { checked } }); - }); - component.update(); + const setSelectedNodeAttribute = (phase: string) => + createFormSetValueAction(`${phase}-selectedNodeAttrs`); + + const setReplicas = async (value: string) => { + await createFormToggleAction('warm-setReplicasSwitch')(true); + await createFormSetValueAction('warm-selectedReplicaCount')(value); }; - const setIndexPriority = (phase: string) => async (value: string) => { - await act(async () => { - find(`${phase}-phaseIndexPriority`).simulate('change', { target: { value } }); - }); - component.update(); + const setShrink = async (value: string) => { + await createFormToggleAction('shrinkSwitch')(true); + await createFormSetValueAction('warm-selectedPrimaryShardCount')(value); }; return { @@ -147,6 +182,20 @@ export const setup = async () => { setBestCompression: setBestCompression('hot'), setIndexPriority: setIndexPriority('hot'), }, + warm: { + enable: enable('warm'), + warmPhaseOnRollover, + setMinAgeValue: setMinAgeValue('warm'), + setMinAgeUnits: setMinAgeUnits('warm'), + setDataAllocation: setDataAllocation('warm'), + setSelectedNodeAttribute: setSelectedNodeAttribute('warm'), + setReplicas, + setShrink, + toggleForceMerge: toggleForceMerge('warm'), + setForcemergeSegments: setForcemergeSegmentsCount('warm'), + setBestCompression: setBestCompression('warm'), + setIndexPriority: setIndexPriority('warm'), + }, }, }; }; diff --git a/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/edit_policy/edit_policy.test.ts b/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/edit_policy/edit_policy.test.ts index 3cbc2d982566e6..fccffde3f793f2 100644 --- a/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/edit_policy/edit_policy.test.ts +++ b/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/edit_policy/edit_policy.test.ts @@ -15,6 +15,9 @@ import { NEW_SNAPSHOT_POLICY_NAME, SNAPSHOT_POLICY_NAME, DEFAULT_POLICY, + POLICY_WITH_INCLUDE_EXCLUDE, + POLICY_WITH_NODE_ATTR_AND_OFF_ALLOCATION, + POLICY_WITH_NODE_ROLE_ALLOCATION, } from './constants'; window.scrollTo = jest.fn(); @@ -53,7 +56,8 @@ describe('', () => { await actions.savePolicy(); const latestRequest = server.requests[server.requests.length - 1]; - expect(JSON.parse(JSON.parse(latestRequest.requestBody).body)).toMatchInlineSnapshot(` + const entirePolicy = JSON.parse(JSON.parse(latestRequest.requestBody).body); + expect(entirePolicy).toMatchInlineSnapshot(` Object { "name": "my_policy", "phases": Object { @@ -81,21 +85,184 @@ describe('', () => { test('disabling rollover', async () => { const { actions } = testBed; - await actions.hot.toggleRollover(false); + await actions.hot.toggleRollover(true); await actions.savePolicy(); const latestRequest = server.requests[server.requests.length - 1]; - expect(JSON.parse(JSON.parse(latestRequest.requestBody).body)).toMatchInlineSnapshot(` + const policy = JSON.parse(JSON.parse(latestRequest.requestBody).body); + const hotActions = policy.phases.hot.actions; + const rolloverAction = hotActions.rollover; + expect(rolloverAction).toBe(undefined); + expect(hotActions).toMatchInlineSnapshot(` + Object { + "set_priority": Object { + "priority": 100, + }, + } + `); + }); + }); + }); + + describe('warm phase', () => { + describe('serialization', () => { + beforeEach(async () => { + httpRequestsMockHelpers.setLoadPolicies([DEFAULT_POLICY]); + httpRequestsMockHelpers.setListNodes({ + nodesByRoles: {}, + nodesByAttributes: { test: ['123'] }, + isUsingDeprecatedDataRoleConfig: false, + }); + httpRequestsMockHelpers.setLoadSnapshotPolicies([]); + + await act(async () => { + testBed = await setup(); + }); + + const { component } = testBed; + component.update(); + }); + + test('default values', async () => { + const { actions } = testBed; + await actions.warm.enable(true); + await actions.savePolicy(); + const latestRequest = server.requests[server.requests.length - 1]; + const warmPhase = JSON.parse(JSON.parse(latestRequest.requestBody).body).phases.warm; + expect(warmPhase).toMatchInlineSnapshot(` + Object { + "actions": Object { + "set_priority": Object { + "priority": 50, + }, + }, + "min_age": "0ms", + } + `); + }); + + test('setting all values', async () => { + const { actions } = testBed; + await actions.warm.enable(true); + await actions.warm.setMinAgeValue('123'); + await actions.warm.setMinAgeUnits('d'); + await actions.warm.setDataAllocation('node_attrs'); + await actions.warm.setSelectedNodeAttribute('test:123'); + await actions.warm.setReplicas('123'); + await actions.warm.setShrink('123'); + await actions.warm.toggleForceMerge(true); + await actions.warm.setForcemergeSegments('123'); + await actions.warm.setBestCompression(true); + await actions.warm.setIndexPriority('123'); + await actions.savePolicy(); + const latestRequest = server.requests[server.requests.length - 1]; + const entirePolicy = JSON.parse(JSON.parse(latestRequest.requestBody).body); + // Check shape of entire policy + expect(entirePolicy).toMatchInlineSnapshot(` Object { "name": "my_policy", "phases": Object { "hot": Object { "actions": Object { + "rollover": Object { + "max_age": "30d", + "max_size": "50gb", + }, "set_priority": Object { "priority": 100, }, }, "min_age": "0ms", }, + "warm": Object { + "actions": Object { + "allocate": Object { + "number_of_replicas": 123, + "require": Object { + "test": "123", + }, + }, + "forcemerge": Object { + "index_codec": "best_compression", + "max_num_segments": 123, + }, + "set_priority": Object { + "priority": 123, + }, + "shrink": Object { + "number_of_shards": 123, + }, + }, + "min_age": "123d", + }, + }, + } + `); + }); + + test('default allocation with replicas set', async () => { + const { actions } = testBed; + await actions.warm.enable(true); + await actions.warm.setReplicas('123'); + await actions.savePolicy(); + const latestRequest = server.requests[server.requests.length - 1]; + const warmPhaseActions = JSON.parse(JSON.parse(latestRequest.requestBody).body).phases.warm + .actions; + expect(warmPhaseActions).toMatchInlineSnapshot(` + Object { + "allocate": Object { + "number_of_replicas": 123, + }, + "set_priority": Object { + "priority": 50, + }, + } + `); + }); + + test('setting warm phase on rollover to "true"', async () => { + const { actions } = testBed; + await actions.warm.enable(true); + await actions.warm.warmPhaseOnRollover(true); + await actions.savePolicy(); + const latestRequest = server.requests[server.requests.length - 1]; + const warmPhaseMinAge = JSON.parse(JSON.parse(latestRequest.requestBody).body).phases.warm + .min_age; + expect(warmPhaseMinAge).toBe(undefined); + }); + }); + + describe('policy with include and exclude', () => { + beforeEach(async () => { + httpRequestsMockHelpers.setLoadPolicies([POLICY_WITH_INCLUDE_EXCLUDE]); + httpRequestsMockHelpers.setListNodes({ + nodesByRoles: {}, + nodesByAttributes: { test: ['123'] }, + isUsingDeprecatedDataRoleConfig: false, + }); + httpRequestsMockHelpers.setLoadSnapshotPolicies([]); + + await act(async () => { + testBed = await setup(); + }); + + const { component } = testBed; + component.update(); + }); + + test('preserves include, exclude allocation settings', async () => { + const { actions } = testBed; + await actions.warm.setDataAllocation('node_attrs'); + await actions.savePolicy(); + const latestRequest = server.requests[server.requests.length - 1]; + const warmPhaseAllocate = JSON.parse(JSON.parse(latestRequest.requestBody).body).phases.warm + .actions.allocate; + expect(warmPhaseAllocate).toMatchInlineSnapshot(` + Object { + "exclude": Object { + "def": "456", + }, + "include": Object { + "abc": "123", }, } `); @@ -216,4 +383,57 @@ describe('', () => { expect(testBed.find('policiesErrorCallout').exists()).toBeTruthy(); }); }); + + describe('data allocation', () => { + describe('node roles', () => { + beforeEach(async () => { + httpRequestsMockHelpers.setLoadPolicies([POLICY_WITH_NODE_ROLE_ALLOCATION]); + httpRequestsMockHelpers.setListNodes({ + isUsingDeprecatedDataRoleConfig: false, + nodesByAttributes: { test: ['123'] }, + nodesByRoles: { data: ['123'] }, + }); + + await act(async () => { + testBed = await setup(); + }); + + const { component } = testBed; + component.update(); + }); + test('showing "default" type', () => { + const { find } = testBed; + expect(find('warm-dataTierAllocationControls.dataTierSelect').text()).toContain( + 'recommended' + ); + expect(find('warm-dataTierAllocationControls.dataTierSelect').text()).not.toContain( + 'Custom' + ); + expect(find('warm-dataTierAllocationControls.dataTierSelect').text()).not.toContain('Off'); + }); + }); + describe('node attr and none', () => { + beforeEach(async () => { + httpRequestsMockHelpers.setLoadPolicies([POLICY_WITH_NODE_ATTR_AND_OFF_ALLOCATION]); + httpRequestsMockHelpers.setListNodes({ + isUsingDeprecatedDataRoleConfig: false, + nodesByAttributes: { test: ['123'] }, + nodesByRoles: { data: ['123'] }, + }); + + await act(async () => { + testBed = await setup(); + }); + + const { component } = testBed; + component.update(); + }); + + test('showing "custom" and "off" types', () => { + const { find } = testBed; + expect(find('warm-dataTierAllocationControls.dataTierSelect').text()).toContain('Custom'); + expect(find('cold-dataTierAllocationControls.dataTierSelect').text()).toContain('Off'); + }); + }); + }); }); diff --git a/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/helpers/http_requests.ts b/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/helpers/http_requests.ts index 04f58f93939ca3..c7a493ce80d96b 100644 --- a/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/helpers/http_requests.ts +++ b/x-pack/plugins/index_lifecycle_management/__jest__/client_integration/helpers/http_requests.ts @@ -6,6 +6,7 @@ import { fakeServer, SinonFakeServer } from 'sinon'; import { API_BASE_PATH } from '../../../common/constants'; +import { ListNodesRouteResponse } from '../../../common/types'; export const init = () => { const server = fakeServer.create(); @@ -38,8 +39,17 @@ const registerHttpRequestMockHelpers = (server: SinonFakeServer) => { ]); }; + const setListNodes = (body: ListNodesRouteResponse) => { + server.respondWith('GET', `${API_BASE_PATH}/nodes/list`, [ + 200, + { 'Content-Type': 'application/json' }, + JSON.stringify(body), + ]); + }; + return { setLoadPolicies, setLoadSnapshotPolicies, + setListNodes, }; }; diff --git a/x-pack/plugins/index_lifecycle_management/__jest__/components/edit_policy.test.tsx b/x-pack/plugins/index_lifecycle_management/__jest__/components/edit_policy.test.tsx index c7664f2d837b98..4ba6cee7b027f3 100644 --- a/x-pack/plugins/index_lifecycle_management/__jest__/components/edit_policy.test.tsx +++ b/x-pack/plugins/index_lifecycle_management/__jest__/components/edit_policy.test.tsx @@ -30,9 +30,7 @@ import { init as initUiMetric } from '../../public/application/services/ui_metri import { init as initNotification } from '../../public/application/services/notification'; import { PolicyFromES } from '../../common/types'; import { - positiveNumbersAboveZeroErrorMessage, positiveNumberRequiredMessage, - numberRequiredMessage, policyNameRequiredMessage, policyNameStartsWithUnderscoreErrorMessage, policyNameContainsCommaErrorMessage, @@ -40,6 +38,8 @@ import { policyNameMustBeDifferentErrorMessage, policyNameAlreadyUsedErrorMessage, } from '../../public/application/services/policies/policy_validation'; + +import { i18nTexts } from '../../public/application/sections/edit_policy/i18n_texts'; import { editPolicyHelpers } from './helpers'; // @ts-ignore @@ -89,13 +89,13 @@ const activatePhase = async (rendered: ReactWrapper, phase: string) => { }); rendered.update(); }; -const openNodeAttributesSection = (rendered: ReactWrapper, phase: string) => { +const openNodeAttributesSection = async (rendered: ReactWrapper, phase: string) => { const getControls = () => findTestSubject(rendered, `${phase}-dataTierAllocationControls`); - act(() => { + await act(async () => { findTestSubject(getControls(), 'dataTierSelect').simulate('click'); }); rendered.update(); - act(() => { + await act(async () => { findTestSubject(getControls(), 'customDataAllocationOption').simulate('click'); }); rendered.update(); @@ -119,19 +119,29 @@ const noRollover = async (rendered: ReactWrapper) => { }); rendered.update(); }; -const getNodeAttributeSelect = (rendered: ReactWrapper, phase: string) => { +const getNodeAttributeSelectLegacy = (rendered: ReactWrapper, phase: string) => { return rendered.find(`select#${phase}-selectedNodeAttrs`); }; +const getNodeAttributeSelect = (rendered: ReactWrapper, phase: string) => { + return findTestSubject(rendered, `${phase}-selectedNodeAttrs`); +}; const setPolicyName = (rendered: ReactWrapper, policyName: string) => { const policyNameField = findTestSubject(rendered, 'policyNameField'); policyNameField.simulate('change', { target: { value: policyName } }); rendered.update(); }; -const setPhaseAfter = (rendered: ReactWrapper, phase: string, after: string | number) => { +const setPhaseAfterLegacy = (rendered: ReactWrapper, phase: string, after: string | number) => { const afterInput = rendered.find(`input#${phase}-selectedMinimumAge`); afterInput.simulate('change', { target: { value: after } }); rendered.update(); }; +const setPhaseAfter = async (rendered: ReactWrapper, phase: string, after: string | number) => { + const afterInput = findTestSubject(rendered, `${phase}-selectedMinimumAge`); + await act(async () => { + afterInput.simulate('change', { target: { value: after } }); + }); + rendered.update(); +}; const setPhaseIndexPriorityLegacy = ( rendered: ReactWrapper, phase: string, @@ -172,10 +182,11 @@ describe('edit policy', () => { * any validation errors. This helper advances timers and can trigger component * state changes. */ - const waitForFormLibValidation = () => { + const waitForFormLibValidation = (rendered: ReactWrapper) => { act(() => { jest.advanceTimersByTime(1000); }); + rendered.update(); }; beforeEach(() => { @@ -288,13 +299,12 @@ describe('edit policy', () => { await act(async () => { maxSizeInput.simulate('change', { target: { value: '' } }); }); - waitForFormLibValidation(); + waitForFormLibValidation(rendered); const maxAgeInput = findTestSubject(rendered, 'hot-selectedMaxAge'); await act(async () => { maxAgeInput.simulate('change', { target: { value: '' } }); }); - waitForFormLibValidation(); - rendered.update(); + waitForFormLibValidation(rendered); await save(rendered); expect(findTestSubject(rendered, 'rolloverSettingsRequired').exists()).toBeTruthy(); }); @@ -305,9 +315,9 @@ describe('edit policy', () => { await act(async () => { maxSizeInput.simulate('change', { target: { value: '-1' } }); }); - waitForFormLibValidation(); + waitForFormLibValidation(rendered); rendered.update(); - expectedErrorMessages(rendered, [positiveNumbersAboveZeroErrorMessage]); + expectedErrorMessages(rendered, [i18nTexts.editPolicy.errors.numberGreatThan0Required]); }); test('should show number above 0 required error when trying to save with 0 for max size', async () => { const rendered = mountWithIntl(component); @@ -316,9 +326,8 @@ describe('edit policy', () => { await act(async () => { maxSizeInput.simulate('change', { target: { value: '-1' } }); }); - waitForFormLibValidation(); - rendered.update(); - expectedErrorMessages(rendered, [positiveNumbersAboveZeroErrorMessage]); + waitForFormLibValidation(rendered); + expectedErrorMessages(rendered, [i18nTexts.editPolicy.errors.numberGreatThan0Required]); }); test('should show number above 0 required error when trying to save with -1 for max age', async () => { const rendered = mountWithIntl(component); @@ -327,9 +336,8 @@ describe('edit policy', () => { await act(async () => { maxAgeInput.simulate('change', { target: { value: '-1' } }); }); - waitForFormLibValidation(); - rendered.update(); - expectedErrorMessages(rendered, [positiveNumbersAboveZeroErrorMessage]); + waitForFormLibValidation(rendered); + expectedErrorMessages(rendered, [i18nTexts.editPolicy.errors.numberGreatThan0Required]); }); test('should show number above 0 required error when trying to save with 0 for max age', async () => { const rendered = mountWithIntl(component); @@ -338,9 +346,8 @@ describe('edit policy', () => { await act(async () => { maxAgeInput.simulate('change', { target: { value: '0' } }); }); - waitForFormLibValidation(); - rendered.update(); - expectedErrorMessages(rendered, [positiveNumbersAboveZeroErrorMessage]); + waitForFormLibValidation(rendered); + expectedErrorMessages(rendered, [i18nTexts.editPolicy.errors.numberGreatThan0Required]); }); test('should show forcemerge input when rollover enabled', () => { const rendered = mountWithIntl(component); @@ -351,8 +358,7 @@ describe('edit policy', () => { const rendered = mountWithIntl(component); setPolicyName(rendered, 'mypolicy'); await noRollover(rendered); - waitForFormLibValidation(); - rendered.update(); + waitForFormLibValidation(rendered); expect(findTestSubject(rendered, 'hot-forceMergeSwitch').exists()).toBeFalsy(); }); test('should show positive number required above zero error when trying to save hot phase with 0 for force merge', async () => { @@ -366,9 +372,8 @@ describe('edit policy', () => { await act(async () => { forcemergeInput.simulate('change', { target: { value: '0' } }); }); - waitForFormLibValidation(); - rendered.update(); - expectedErrorMessages(rendered, [positiveNumbersAboveZeroErrorMessage]); + waitForFormLibValidation(rendered); + expectedErrorMessages(rendered, [i18nTexts.editPolicy.errors.numberGreatThan0Required]); }); test('should show positive number above 0 required error when trying to save hot phase with -1 for force merge', async () => { const rendered = mountWithIntl(component); @@ -379,19 +384,17 @@ describe('edit policy', () => { await act(async () => { forcemergeInput.simulate('change', { target: { value: '-1' } }); }); - waitForFormLibValidation(); - rendered.update(); + waitForFormLibValidation(rendered); await save(rendered); - expectedErrorMessages(rendered, [positiveNumbersAboveZeroErrorMessage]); + expectedErrorMessages(rendered, [i18nTexts.editPolicy.errors.numberGreatThan0Required]); }); test('should show positive number required error when trying to save with -1 for index priority', async () => { const rendered = mountWithIntl(component); await noRollover(rendered); setPolicyName(rendered, 'mypolicy'); await setPhaseIndexPriority(rendered, 'hot', '-1'); - waitForFormLibValidation(); - rendered.update(); - expectedErrorMessages(rendered, [positiveNumbersAboveZeroErrorMessage]); + waitForFormLibValidation(rendered); + expectedErrorMessages(rendered, [i18nTexts.editPolicy.errors.numberGreatThan0Required]); }); }); describe('warm phase', () => { @@ -408,17 +411,17 @@ describe('edit policy', () => { await noRollover(rendered); setPolicyName(rendered, 'mypolicy'); await activatePhase(rendered, 'warm'); - setPhaseAfter(rendered, 'warm', ''); - await save(rendered); - expectedErrorMessages(rendered, [numberRequiredMessage]); + await setPhaseAfter(rendered, 'warm', ''); + waitForFormLibValidation(rendered); + expectedErrorMessages(rendered, [i18nTexts.editPolicy.errors.nonNegativeNumberRequired]); }); test('should allow 0 for phase timing', async () => { const rendered = mountWithIntl(component); await noRollover(rendered); setPolicyName(rendered, 'mypolicy'); await activatePhase(rendered, 'warm'); - setPhaseAfter(rendered, 'warm', '0'); - await save(rendered); + await setPhaseAfter(rendered, 'warm', '0'); + waitForFormLibValidation(rendered); expectedErrorMessages(rendered, []); }); test('should show positive number required error when trying to save warm phase with -1 for after', async () => { @@ -426,75 +429,87 @@ describe('edit policy', () => { await noRollover(rendered); setPolicyName(rendered, 'mypolicy'); await activatePhase(rendered, 'warm'); - setPhaseAfter(rendered, 'warm', '-1'); - await save(rendered); - expectedErrorMessages(rendered, [positiveNumberRequiredMessage]); + await setPhaseAfter(rendered, 'warm', '-1'); + waitForFormLibValidation(rendered); + expectedErrorMessages(rendered, [i18nTexts.editPolicy.errors.nonNegativeNumberRequired]); }); test('should show positive number required error when trying to save warm phase with -1 for index priority', async () => { const rendered = mountWithIntl(component); await noRollover(rendered); setPolicyName(rendered, 'mypolicy'); await activatePhase(rendered, 'warm'); - setPhaseAfter(rendered, 'warm', '1'); - setPhaseIndexPriorityLegacy(rendered, 'warm', '-1'); - await save(rendered); - expectedErrorMessages(rendered, [positiveNumberRequiredMessage]); + await setPhaseAfter(rendered, 'warm', '1'); + await setPhaseAfter(rendered, 'warm', '-1'); + waitForFormLibValidation(rendered); + expectedErrorMessages(rendered, [i18nTexts.editPolicy.errors.nonNegativeNumberRequired]); }); test('should show positive number required above zero error when trying to save warm phase with 0 for shrink', async () => { const rendered = mountWithIntl(component); await noRollover(rendered); setPolicyName(rendered, 'mypolicy'); await activatePhase(rendered, 'warm'); - findTestSubject(rendered, 'shrinkSwitch').simulate('click'); - rendered.update(); - setPhaseAfter(rendered, 'warm', '1'); - const shrinkInput = rendered.find('input#warm-selectedPrimaryShardCount'); - shrinkInput.simulate('change', { target: { value: '0' } }); + act(() => { + findTestSubject(rendered, 'shrinkSwitch').simulate('click'); + }); rendered.update(); - await save(rendered); - expectedErrorMessages(rendered, [positiveNumbersAboveZeroErrorMessage]); + await setPhaseAfter(rendered, 'warm', '1'); + const shrinkInput = findTestSubject(rendered, 'warm-selectedPrimaryShardCount'); + await act(async () => { + shrinkInput.simulate('change', { target: { value: '0' } }); + }); + waitForFormLibValidation(rendered); + expectedErrorMessages(rendered, [i18nTexts.editPolicy.errors.numberGreatThan0Required]); }); test('should show positive number above 0 required error when trying to save warm phase with -1 for shrink', async () => { const rendered = mountWithIntl(component); await noRollover(rendered); setPolicyName(rendered, 'mypolicy'); await activatePhase(rendered, 'warm'); - setPhaseAfter(rendered, 'warm', '1'); - findTestSubject(rendered, 'shrinkSwitch').simulate('click'); - rendered.update(); - const shrinkInput = rendered.find('input#warm-selectedPrimaryShardCount'); - shrinkInput.simulate('change', { target: { value: '-1' } }); + await setPhaseAfter(rendered, 'warm', '1'); + act(() => { + findTestSubject(rendered, 'shrinkSwitch').simulate('click'); + }); rendered.update(); - await save(rendered); - expectedErrorMessages(rendered, [positiveNumbersAboveZeroErrorMessage]); + const shrinkInput = findTestSubject(rendered, 'warm-selectedPrimaryShardCount'); + await act(async () => { + shrinkInput.simulate('change', { target: { value: '-1' } }); + }); + waitForFormLibValidation(rendered); + expectedErrorMessages(rendered, [i18nTexts.editPolicy.errors.numberGreatThan0Required]); }); test('should show positive number required above zero error when trying to save warm phase with 0 for force merge', async () => { const rendered = mountWithIntl(component); await noRollover(rendered); setPolicyName(rendered, 'mypolicy'); await activatePhase(rendered, 'warm'); - setPhaseAfter(rendered, 'warm', '1'); - findTestSubject(rendered, 'warm-forceMergeSwitch').simulate('click'); + await setPhaseAfter(rendered, 'warm', '1'); + act(() => { + findTestSubject(rendered, 'warm-forceMergeSwitch').simulate('click'); + }); rendered.update(); const forcemergeInput = findTestSubject(rendered, 'warm-selectedForceMergeSegments'); - forcemergeInput.simulate('change', { target: { value: '0' } }); - rendered.update(); - await save(rendered); - expectedErrorMessages(rendered, [positiveNumbersAboveZeroErrorMessage]); + await act(async () => { + forcemergeInput.simulate('change', { target: { value: '0' } }); + }); + waitForFormLibValidation(rendered); + expectedErrorMessages(rendered, [i18nTexts.editPolicy.errors.numberGreatThan0Required]); }); test('should show positive number above 0 required error when trying to save warm phase with -1 for force merge', async () => { const rendered = mountWithIntl(component); await noRollover(rendered); setPolicyName(rendered, 'mypolicy'); await activatePhase(rendered, 'warm'); - setPhaseAfter(rendered, 'warm', '1'); - findTestSubject(rendered, 'warm-forceMergeSwitch').simulate('click'); + await setPhaseAfter(rendered, 'warm', '1'); + await act(async () => { + findTestSubject(rendered, 'warm-forceMergeSwitch').simulate('click'); + }); rendered.update(); const forcemergeInput = findTestSubject(rendered, 'warm-selectedForceMergeSegments'); - forcemergeInput.simulate('change', { target: { value: '-1' } }); - rendered.update(); - await save(rendered); - expectedErrorMessages(rendered, [positiveNumbersAboveZeroErrorMessage]); + await act(async () => { + forcemergeInput.simulate('change', { target: { value: '-1' } }); + }); + waitForFormLibValidation(rendered); + expectedErrorMessages(rendered, [i18nTexts.editPolicy.errors.numberGreatThan0Required]); }); test('should show spinner for node attributes input when loading', async () => { server.respondImmediately = false; @@ -504,7 +519,7 @@ describe('edit policy', () => { await activatePhase(rendered, 'warm'); expect(rendered.find('.euiLoadingSpinner').exists()).toBeTruthy(); expect(rendered.find('.euiCallOut--warning').exists()).toBeFalsy(); - expect(getNodeAttributeSelect(rendered, 'warm').exists()).toBeFalsy(); + expect(getNodeAttributeSelectLegacy(rendered, 'warm').exists()).toBeFalsy(); }); test('should show warning instead of node attributes input when none exist', async () => { http.setupNodeListResponse({ @@ -517,9 +532,9 @@ describe('edit policy', () => { setPolicyName(rendered, 'mypolicy'); await activatePhase(rendered, 'warm'); expect(rendered.find('.euiLoadingSpinner').exists()).toBeFalsy(); - openNodeAttributesSection(rendered, 'warm'); + await openNodeAttributesSection(rendered, 'warm'); expect(findTestSubject(rendered, 'noNodeAttributesWarning').exists()).toBeTruthy(); - expect(getNodeAttributeSelect(rendered, 'warm').exists()).toBeFalsy(); + expect(getNodeAttributeSelectLegacy(rendered, 'warm').exists()).toBeFalsy(); }); test('should show node attributes input when attributes exist', async () => { const rendered = mountWithIntl(component); @@ -527,7 +542,7 @@ describe('edit policy', () => { setPolicyName(rendered, 'mypolicy'); await activatePhase(rendered, 'warm'); expect(rendered.find('.euiLoadingSpinner').exists()).toBeFalsy(); - openNodeAttributesSection(rendered, 'warm'); + await openNodeAttributesSection(rendered, 'warm'); expect(findTestSubject(rendered, 'noNodeAttributesWarning').exists()).toBeFalsy(); const nodeAttributesSelect = getNodeAttributeSelect(rendered, 'warm'); expect(nodeAttributesSelect.exists()).toBeTruthy(); @@ -539,13 +554,15 @@ describe('edit policy', () => { setPolicyName(rendered, 'mypolicy'); await activatePhase(rendered, 'warm'); expect(rendered.find('.euiLoadingSpinner').exists()).toBeFalsy(); - openNodeAttributesSection(rendered, 'warm'); + await openNodeAttributesSection(rendered, 'warm'); expect(findTestSubject(rendered, 'noNodeAttributesWarning').exists()).toBeFalsy(); const nodeAttributesSelect = getNodeAttributeSelect(rendered, 'warm'); expect(nodeAttributesSelect.exists()).toBeTruthy(); expect(findTestSubject(rendered, 'warm-viewNodeDetailsFlyoutButton').exists()).toBeFalsy(); expect(nodeAttributesSelect.find('option').length).toBe(2); - nodeAttributesSelect.simulate('change', { target: { value: 'attribute:true' } }); + await act(async () => { + nodeAttributesSelect.simulate('change', { target: { value: 'attribute:true' } }); + }); rendered.update(); const flyoutButton = findTestSubject(rendered, 'warm-viewNodeDetailsFlyoutButton'); expect(flyoutButton.exists()).toBeTruthy(); @@ -608,7 +625,7 @@ describe('edit policy', () => { await noRollover(rendered); setPolicyName(rendered, 'mypolicy'); await activatePhase(rendered, 'cold'); - setPhaseAfter(rendered, 'cold', '0'); + setPhaseAfterLegacy(rendered, 'cold', '0'); await save(rendered); expectedErrorMessages(rendered, []); }); @@ -617,7 +634,7 @@ describe('edit policy', () => { await noRollover(rendered); setPolicyName(rendered, 'mypolicy'); await activatePhase(rendered, 'cold'); - setPhaseAfter(rendered, 'cold', '-1'); + setPhaseAfterLegacy(rendered, 'cold', '-1'); await save(rendered); expectedErrorMessages(rendered, [positiveNumberRequiredMessage]); }); @@ -629,7 +646,7 @@ describe('edit policy', () => { await activatePhase(rendered, 'cold'); expect(rendered.find('.euiLoadingSpinner').exists()).toBeTruthy(); expect(rendered.find('.euiCallOut--warning').exists()).toBeFalsy(); - expect(getNodeAttributeSelect(rendered, 'cold').exists()).toBeFalsy(); + expect(getNodeAttributeSelectLegacy(rendered, 'cold').exists()).toBeFalsy(); }); test('should show warning instead of node attributes input when none exist', async () => { http.setupNodeListResponse({ @@ -642,9 +659,9 @@ describe('edit policy', () => { setPolicyName(rendered, 'mypolicy'); await activatePhase(rendered, 'cold'); expect(rendered.find('.euiLoadingSpinner').exists()).toBeFalsy(); - openNodeAttributesSection(rendered, 'cold'); + await openNodeAttributesSection(rendered, 'cold'); expect(findTestSubject(rendered, 'noNodeAttributesWarning').exists()).toBeTruthy(); - expect(getNodeAttributeSelect(rendered, 'cold').exists()).toBeFalsy(); + expect(getNodeAttributeSelectLegacy(rendered, 'cold').exists()).toBeFalsy(); }); test('should show node attributes input when attributes exist', async () => { const rendered = mountWithIntl(component); @@ -652,9 +669,9 @@ describe('edit policy', () => { setPolicyName(rendered, 'mypolicy'); await activatePhase(rendered, 'cold'); expect(rendered.find('.euiLoadingSpinner').exists()).toBeFalsy(); - openNodeAttributesSection(rendered, 'cold'); + await openNodeAttributesSection(rendered, 'cold'); expect(findTestSubject(rendered, 'noNodeAttributesWarning').exists()).toBeFalsy(); - const nodeAttributesSelect = getNodeAttributeSelect(rendered, 'cold'); + const nodeAttributesSelect = getNodeAttributeSelectLegacy(rendered, 'cold'); expect(nodeAttributesSelect.exists()).toBeTruthy(); expect(nodeAttributesSelect.find('option').length).toBe(2); }); @@ -664,9 +681,9 @@ describe('edit policy', () => { setPolicyName(rendered, 'mypolicy'); await activatePhase(rendered, 'cold'); expect(rendered.find('.euiLoadingSpinner').exists()).toBeFalsy(); - openNodeAttributesSection(rendered, 'cold'); + await openNodeAttributesSection(rendered, 'cold'); expect(findTestSubject(rendered, 'noNodeAttributesWarning').exists()).toBeFalsy(); - const nodeAttributesSelect = getNodeAttributeSelect(rendered, 'cold'); + const nodeAttributesSelect = getNodeAttributeSelectLegacy(rendered, 'cold'); expect(nodeAttributesSelect.exists()).toBeTruthy(); expect(findTestSubject(rendered, 'cold-viewNodeDetailsFlyoutButton').exists()).toBeFalsy(); expect(nodeAttributesSelect.find('option').length).toBe(2); @@ -685,7 +702,7 @@ describe('edit policy', () => { await noRollover(rendered); setPolicyName(rendered, 'mypolicy'); await activatePhase(rendered, 'cold'); - setPhaseAfter(rendered, 'cold', '1'); + setPhaseAfterLegacy(rendered, 'cold', '1'); setPhaseIndexPriorityLegacy(rendered, 'cold', '-1'); await save(rendered); expectedErrorMessages(rendered, [positiveNumberRequiredMessage]); @@ -736,7 +753,7 @@ describe('edit policy', () => { await noRollover(rendered); setPolicyName(rendered, 'mypolicy'); await activatePhase(rendered, 'delete'); - setPhaseAfter(rendered, 'delete', '0'); + setPhaseAfterLegacy(rendered, 'delete', '0'); await save(rendered); expectedErrorMessages(rendered, []); }); @@ -745,7 +762,7 @@ describe('edit policy', () => { await noRollover(rendered); setPolicyName(rendered, 'mypolicy'); await activatePhase(rendered, 'delete'); - setPhaseAfter(rendered, 'delete', '-1'); + setPhaseAfterLegacy(rendered, 'delete', '-1'); await save(rendered); expectedErrorMessages(rendered, [positiveNumberRequiredMessage]); }); @@ -796,7 +813,8 @@ describe('edit policy', () => { test('should hide data tier option on cloud using legacy node role configuration', async () => { http.setupNodeListResponse({ nodesByAttributes: { test: ['123'] }, - nodesByRoles: { data: ['test'], data_hot: ['test'], data_warm: ['test'] }, + // On cloud, if using legacy config there will not be any "data_*" roles set. + nodesByRoles: { data: ['test'] }, isUsingDeprecatedDataRoleConfig: true, }); const rendered = mountWithIntl(component); diff --git a/x-pack/plugins/index_lifecycle_management/common/constants/data_tiers.ts b/x-pack/plugins/index_lifecycle_management/common/constants/data_tiers.ts index 8a1acf72949e68..f6b9506f9068cc 100644 --- a/x-pack/plugins/index_lifecycle_management/common/constants/data_tiers.ts +++ b/x-pack/plugins/index_lifecycle_management/common/constants/data_tiers.ts @@ -6,13 +6,13 @@ // Order of node roles matters here, the warm phase prefers allocating data // to the data_warm role. -import { NodeDataRole, PhaseWithAllocation } from '../types'; +import { DataTierRole, PhaseWithAllocation } from '../types'; -const WARM_PHASE_NODE_PREFERENCE: NodeDataRole[] = ['data_warm', 'data_hot']; +const WARM_PHASE_NODE_PREFERENCE: DataTierRole[] = ['data_warm', 'data_hot']; -const COLD_PHASE_NODE_PREFERENCE: NodeDataRole[] = ['data_cold', 'data_warm', 'data_hot']; +const COLD_PHASE_NODE_PREFERENCE: DataTierRole[] = ['data_cold', 'data_warm', 'data_hot']; -export const phaseToNodePreferenceMap: Record = Object.freeze({ +export const phaseToNodePreferenceMap: Record = Object.freeze({ warm: WARM_PHASE_NODE_PREFERENCE, cold: COLD_PHASE_NODE_PREFERENCE, }); diff --git a/x-pack/plugins/index_lifecycle_management/common/types/api.ts b/x-pack/plugins/index_lifecycle_management/common/types/api.ts index ccdd7fcb11778d..b7ca16ac46ddef 100644 --- a/x-pack/plugins/index_lifecycle_management/common/types/api.ts +++ b/x-pack/plugins/index_lifecycle_management/common/types/api.ts @@ -4,11 +4,11 @@ * you may not use this file except in compliance with the Elastic License. */ -import { NodeDataRoleWithCatchAll } from '.'; +import { AnyDataRole } from '.'; export interface ListNodesRouteResponse { nodesByAttributes: { [attributePair: string]: string[] }; - nodesByRoles: { [role in NodeDataRoleWithCatchAll]?: string[] }; + nodesByRoles: { [role in AnyDataRole]?: string[] }; /** * A flag to indicate whether a node is using `settings.node.data` which is the now deprecated way cloud configured diff --git a/x-pack/plugins/index_lifecycle_management/common/types/index.ts b/x-pack/plugins/index_lifecycle_management/common/types/index.ts index 1f41370e48f180..737e5a551aae82 100644 --- a/x-pack/plugins/index_lifecycle_management/common/types/index.ts +++ b/x-pack/plugins/index_lifecycle_management/common/types/index.ts @@ -9,8 +9,17 @@ export * from './api'; export * from './policies'; /** - * These roles reflect how nodes are stratified into different data tiers. The "data" role - * is a catch-all that can be used to store data in any phase. + * These roles reflect how nodes are stratified into different data tiers. */ -export type NodeDataRole = 'data_hot' | 'data_warm' | 'data_cold'; -export type NodeDataRoleWithCatchAll = 'data' | NodeDataRole; +export type DataTierRole = 'data_hot' | 'data_warm' | 'data_cold'; + +/** + * The "data_content" role can store all data the ES stack uses for feature + * functionality like security-related indices. + */ +export type DataRole = 'data_content' | DataTierRole; + +/** + * The "data" role can store data allocated to any tier. + */ +export type AnyDataRole = 'data' | DataRole; diff --git a/x-pack/plugins/index_lifecycle_management/common/types/policies.ts b/x-pack/plugins/index_lifecycle_management/common/types/policies.ts index 152c5e4e9e0d70..813fcd9c253f1f 100644 --- a/x-pack/plugins/index_lifecycle_management/common/types/policies.ts +++ b/x-pack/plugins/index_lifecycle_management/common/types/policies.ts @@ -35,6 +35,19 @@ export interface SerializedPhase { }; } +export interface MigrateAction { + /** + * If enabled is ever set it will probably only be set to `false` because the default value + * for this is `true`. Rather leave unspecified for true when serialising. + */ + enabled: boolean; +} + +export interface SerializedActionWithAllocation { + allocate?: AllocateAction; + migrate?: MigrateAction; +} + export interface SerializedHotPhase extends SerializedPhase { actions: { rollover?: { @@ -59,7 +72,7 @@ export interface SerializedWarmPhase extends SerializedPhase { set_priority?: { priority: number | null; }; - migrate?: { enabled: boolean }; + migrate?: MigrateAction; }; } @@ -70,7 +83,7 @@ export interface SerializedColdPhase extends SerializedPhase { set_priority?: { priority: number | null; }; - migrate?: { enabled: boolean }; + migrate?: MigrateAction; }; } @@ -87,18 +100,11 @@ export interface SerializedDeletePhase extends SerializedPhase { export interface AllocateAction { number_of_replicas?: number; - include: {}; - exclude: {}; + include?: {}; + exclude?: {}; require?: { [attribute: string]: string; }; - migrate?: { - /** - * If enabled is ever set it will only be set to `false` because the default value - * for this is `true`. Rather leave unspecified for true. - */ - enabled: false; - }; } export interface ForcemergeAction { @@ -110,7 +116,6 @@ export interface ForcemergeAction { export interface LegacyPolicy { name: string; phases: { - warm: WarmPhase; cold: ColdPhase; delete: DeletePhase; }; @@ -154,17 +159,6 @@ export interface PhaseWithForcemergeAction { bestCompressionEnabled: boolean; } -export interface WarmPhase - extends CommonPhaseSettings, - PhaseWithMinAge, - PhaseWithAllocationAction, - PhaseWithIndexPriority, - PhaseWithForcemergeAction { - warmPhaseOnRollover: boolean; - shrinkEnabled: boolean; - selectedPrimaryShardCount: string; -} - export interface ColdPhase extends CommonPhaseSettings, PhaseWithMinAge, diff --git a/x-pack/plugins/index_lifecycle_management/public/application/constants/policy.ts b/x-pack/plugins/index_lifecycle_management/public/application/constants/policy.ts index c919331082ec34..136b68727672aa 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/constants/policy.ts +++ b/x-pack/plugins/index_lifecycle_management/public/application/constants/policy.ts @@ -4,16 +4,12 @@ * you may not use this file except in compliance with the Elastic License. */ -import { - SerializedPhase, - ColdPhase, - DeletePhase, - WarmPhase, - SerializedPolicy, -} from '../../../common/types'; +import { SerializedPhase, ColdPhase, DeletePhase, SerializedPolicy } from '../../../common/types'; export const defaultSetPriority: string = '100'; +export const defaultPhaseIndexPriority: string = '50'; + export const defaultPolicy: SerializedPolicy = { name: '', phases: { @@ -28,22 +24,6 @@ export const defaultPolicy: SerializedPolicy = { }, }; -export const defaultNewWarmPhase: WarmPhase = { - phaseEnabled: false, - forceMergeEnabled: false, - selectedForceMergeSegments: '', - bestCompressionEnabled: false, - selectedMinimumAge: '0', - selectedMinimumAgeUnits: 'd', - selectedNodeAttrs: '', - shrinkEnabled: false, - selectedPrimaryShardCount: '', - selectedReplicaCount: '', - warmPhaseOnRollover: true, - phaseIndexPriority: '50', - dataTierAllocationType: 'default', -}; - export const defaultNewColdPhase: ColdPhase = { phaseEnabled: false, selectedMinimumAge: '0', diff --git a/x-pack/plugins/index_lifecycle_management/public/application/lib/data_tiers/determine_allocation_type.ts b/x-pack/plugins/index_lifecycle_management/public/application/lib/data_tiers/determine_allocation_type.ts index 4067ad97fc43bc..20ac439e9964f5 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/lib/data_tiers/determine_allocation_type.ts +++ b/x-pack/plugins/index_lifecycle_management/public/application/lib/data_tiers/determine_allocation_type.ts @@ -4,31 +4,63 @@ * you may not use this file except in compliance with the Elastic License. */ -import { DataTierAllocationType, AllocateAction } from '../../../../common/types'; +import { DataTierAllocationType, AllocateAction, MigrateAction } from '../../../../common/types'; /** * Determine what deserialized state the policy config represents. * * See {@DataTierAllocationType} for more information. */ -export const determineDataTierAllocationType = ( - allocateAction?: AllocateAction +export const determineDataTierAllocationTypeLegacy = ( + actions: { + allocate?: AllocateAction; + migrate?: MigrateAction; + } = {} ): DataTierAllocationType => { - if (!allocateAction) { - return 'default'; - } + const { allocate, migrate } = actions; - if (allocateAction.migrate?.enabled === false) { + if (migrate?.enabled === false) { return 'none'; } + if (!allocate) { + return 'default'; + } + if ( - (allocateAction.require && Object.keys(allocateAction.require).length) || - (allocateAction.include && Object.keys(allocateAction.include).length) || - (allocateAction.exclude && Object.keys(allocateAction.exclude).length) + (allocate.require && Object.keys(allocate.require).length) || + (allocate.include && Object.keys(allocate.include).length) || + (allocate.exclude && Object.keys(allocate.exclude).length) ) { return 'custom'; } return 'default'; }; + +export const determineDataTierAllocationType = ( + actions: { + allocate?: AllocateAction; + migrate?: MigrateAction; + } = {} +) => { + const { allocate, migrate } = actions; + + if (migrate?.enabled === false) { + return 'none'; + } + + if (!allocate) { + return 'node_roles'; + } + + if ( + (allocate.require && Object.keys(allocate.require).length) || + (allocate.include && Object.keys(allocate.include).length) || + (allocate.exclude && Object.keys(allocate.exclude).length) + ) { + return 'node_attrs'; + } + + return 'node_roles'; +}; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/lib/data_tiers/get_available_node_roles_for_phase.ts b/x-pack/plugins/index_lifecycle_management/public/application/lib/data_tiers/get_available_node_roles_for_phase.ts index 6daae57330886d..179de0b1d8c8e4 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/lib/data_tiers/get_available_node_roles_for_phase.ts +++ b/x-pack/plugins/index_lifecycle_management/public/application/lib/data_tiers/get_available_node_roles_for_phase.ts @@ -5,14 +5,14 @@ */ import { - NodeDataRole, + DataTierRole, ListNodesRouteResponse, PhaseWithAllocation, } from '../../../../common/types'; import { phaseToNodePreferenceMap } from '../../../../common/constants'; -export type AllocationNodeRole = NodeDataRole | 'none'; +export type AllocationNodeRole = DataTierRole | 'none'; /** * Given a phase and current cluster node roles, determine which nodes the phase diff --git a/x-pack/plugins/index_lifecycle_management/public/application/lib/data_tiers/index.ts b/x-pack/plugins/index_lifecycle_management/public/application/lib/data_tiers/index.ts index 87f2cbc08ecc0b..ac1aae270628d1 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/lib/data_tiers/index.ts +++ b/x-pack/plugins/index_lifecycle_management/public/application/lib/data_tiers/index.ts @@ -7,3 +7,5 @@ export * from './determine_allocation_type'; export * from './get_available_node_roles_for_phase'; + +export * from './is_node_role_first_preference'; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/lib/data_tiers/is_node_role_first_preference.ts b/x-pack/plugins/index_lifecycle_management/public/application/lib/data_tiers/is_node_role_first_preference.ts index 872efa740b1316..fff640a5eb9575 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/lib/data_tiers/is_node_role_first_preference.ts +++ b/x-pack/plugins/index_lifecycle_management/public/application/lib/data_tiers/is_node_role_first_preference.ts @@ -4,9 +4,9 @@ * you may not use this file except in compliance with the Elastic License. */ -import { NodeDataRole, PhaseWithAllocation } from '../../../../common/types'; +import { DataTierRole, PhaseWithAllocation } from '../../../../common/types'; import { phaseToNodePreferenceMap } from '../../../../common/constants'; -export const isNodeRoleFirstPreference = (phase: PhaseWithAllocation, nodeRole: NodeDataRole) => { +export const isNodeRoleFirstPreference = (phase: PhaseWithAllocation, nodeRole: DataTierRole) => { return phaseToNodePreferenceMap[phase][0] === nodeRole; }; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/data_tier_allocation/default_allocation_notice.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/data_tier_allocation/default_allocation_notice.tsx index 42f9e8494a0b3b..3d0052c69607b4 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/data_tier_allocation/default_allocation_notice.tsx +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/data_tier_allocation/default_allocation_notice.tsx @@ -8,11 +8,11 @@ import { i18n } from '@kbn/i18n'; import React, { FunctionComponent } from 'react'; import { EuiCallOut } from '@elastic/eui'; -import { PhaseWithAllocation, NodeDataRole } from '../../../../../../common/types'; +import { PhaseWithAllocation, DataTierRole } from '../../../../../../common/types'; import { AllocationNodeRole } from '../../../../lib'; -const i18nTextsNodeRoleToDataTier: Record = { +const i18nTextsNodeRoleToDataTier: Record = { data_hot: i18n.translate('xpack.indexLifecycleMgmt.editPolicy.dataTierHotLabel', { defaultMessage: 'hot', }), @@ -31,7 +31,7 @@ const i18nTexts = { 'xpack.indexLifecycleMgmt.warmPhase.dataTier.defaultAllocationNotice.warm.title', { defaultMessage: 'No nodes assigned to the warm tier' } ), - body: (nodeRole: NodeDataRole) => + body: (nodeRole: DataTierRole) => i18n.translate('xpack.indexLifecycleMgmt.warmPhase.dataTier.defaultAllocationNotice.warm', { defaultMessage: 'This policy will move data in the warm phase to {tier} tier nodes instead.', @@ -43,7 +43,7 @@ const i18nTexts = { 'xpack.indexLifecycleMgmt.warmPhase.dataTier.defaultAllocationNotice.cold.title', { defaultMessage: 'No nodes assigned to the cold tier' } ), - body: (nodeRole: NodeDataRole) => + body: (nodeRole: DataTierRole) => i18n.translate('xpack.indexLifecycleMgmt.warmPhase.dataTier.defaultAllocationNotice.cold', { defaultMessage: 'This policy will move data in the cold phase to {tier} tier nodes instead.', diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/forcemerge.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/forcemerge_legacy.tsx similarity index 100% rename from x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/forcemerge.tsx rename to x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/forcemerge_legacy.tsx diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/index.ts b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/index.ts index 04d9a6ef3cf67f..2b774b00b98a9f 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/index.ts +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/index.ts @@ -7,11 +7,11 @@ export { ActiveBadge } from './active_badge'; export { ErrableFormRow } from './form_errors'; export { LearnMoreLink } from './learn_more_link'; -export { MinAgeInput } from './min_age_input'; +export { MinAgeInput } from './min_age_input_legacy'; export { OptionalLabel } from './optional_label'; export { PhaseErrorMessage } from './phase_error_message'; export { PolicyJsonFlyout } from './policy_json_flyout'; -export { SetPriorityInput } from './set_priority_input'; +export { SetPriorityInput } from './set_priority_input_legacy'; export { SnapshotPolicies } from './snapshot_policies'; export { DataTierAllocation, @@ -21,6 +21,6 @@ export { DefaultAllocationNotice, } from './data_tier_allocation'; export { DescribedFormField } from './described_form_field'; -export { Forcemerge } from './forcemerge'; +export { Forcemerge } from './forcemerge_legacy'; export * from './phases'; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/min_age_input.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/min_age_input_legacy.tsx similarity index 100% rename from x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/min_age_input.tsx rename to x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/min_age_input_legacy.tsx diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/cold_phase.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/cold_phase.tsx index 7a22fb5bc1f0c8..da6c358aa67c15 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/cold_phase.tsx +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/cold_phase.tsx @@ -28,7 +28,7 @@ import { DescribedFormField, } from '../'; -import { DataTierAllocationField, useRolloverPath } from './shared'; +import { DataTierAllocationFieldLegacy, useRolloverPath } from './shared'; const i18nTexts = { freezeLabel: i18n.translate('xpack.indexLifecycleMgmt.coldPhase.freezeIndexLabel', { @@ -124,7 +124,7 @@ export const ColdPhase: FunctionComponent = ({ {phaseData.phaseEnabled ? ( {/* Data tier allocation section */} - void }> = ({ - setWarmPhaseOnRollover, -}) => { +export const HotPhase: FunctionComponent = () => { const form = useFormContext(); const [formData] = useFormData({ watch: useRolloverPath, @@ -55,10 +51,6 @@ export const HotPhase: FunctionComponent<{ setWarmPhaseOnRollover: (v: boolean) const isShowingErrors = form.isValid === false; const [showEmptyRolloverFieldsError, setShowEmptyRolloverFieldsError] = useState(false); - useEffect(() => { - setWarmPhaseOnRollover(isRolloverEnabled ?? false); - }, [setWarmPhaseOnRollover, isRolloverEnabled]); - return ( <> -
{i18nTexts.rollOverConfigurationCallout.body}
+
{i18nTexts.editPolicy.errors.rollOverConfigurationCallout.body}
 diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/hot_phase/i18n_texts.ts b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/hot_phase/i18n_texts.ts deleted file mode 100644 index 6423b12b86dd24..00000000000000 --- a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/hot_phase/i18n_texts.ts +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { i18n } from '@kbn/i18n'; - -export const i18nTexts = { - maximumAgeRequiredMessage: i18n.translate( - 'xpack.indexLifecycleMgmt.editPolicy.maximumAgeMissingError', - { - defaultMessage: 'A maximum age is required.', - } - ), - maximumSizeRequiredMessage: i18n.translate( - 'xpack.indexLifecycleMgmt.editPolicy.maximumIndexSizeMissingError', - { - defaultMessage: 'A maximum index size is required.', - } - ), - maximumDocumentsRequiredMessage: i18n.translate( - 'xpack.indexLifecycleMgmt.editPolicy.maximumDocumentsMissingError', - { - defaultMessage: 'Maximum documents is required.', - } - ), - rollOverConfigurationCallout: { - title: i18n.translate('xpack.indexLifecycleMgmt.editPolicy.rolloverConfigurationError.title', { - defaultMessage: 'Invalid rollover configuration', - }), - body: i18n.translate('xpack.indexLifecycleMgmt.editPolicy.rolloverConfigurationError.body', { - defaultMessage: - 'A value for one of maximum size, maximum documents, or maximum age is required.', - }), - }, -}; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/components/cloud_data_tier_callout.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/components/cloud_data_tier_callout.tsx new file mode 100644 index 00000000000000..2dff55ac10de18 --- /dev/null +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/components/cloud_data_tier_callout.tsx @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { i18n } from '@kbn/i18n'; +import React, { FunctionComponent } from 'react'; +import { EuiCallOut } from '@elastic/eui'; + +const i18nTexts = { + title: i18n.translate('xpack.indexLifecycleMgmt.editPolicy.cloudDataTierCallout.title', { + defaultMessage: 'Create a cold tier', + }), + body: i18n.translate('xpack.indexLifecycleMgmt.editPolicy.cloudDataTierCallout.body', { + defaultMessage: 'Edit your Elastic Cloud deployment to set up a cold tier.', + }), +}; + +export const CloudDataTierCallout: FunctionComponent = () => { + return ( + + {i18nTexts.body} + + ); +}; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/components/data_tier_allocation.scss b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/components/data_tier_allocation.scss new file mode 100644 index 00000000000000..62ec3f303e1e8c --- /dev/null +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/components/data_tier_allocation.scss @@ -0,0 +1,9 @@ +.indexLifecycleManagement__phase__dataTierAllocation { + &__controlSection { + background-color: $euiColorLightestShade; + padding-top: $euiSizeM; + padding-left: $euiSizeM; + padding-right: $euiSizeM; + padding-bottom: $euiSizeM; + } +} diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/components/data_tier_allocation.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/components/data_tier_allocation.tsx new file mode 100644 index 00000000000000..56af4bbac7a261 --- /dev/null +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/components/data_tier_allocation.tsx @@ -0,0 +1,191 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { FunctionComponent } from 'react'; +import { get } from 'lodash'; +import { i18n } from '@kbn/i18n'; +import { EuiText, EuiSpacer, EuiSuperSelectOption } from '@elastic/eui'; + +import { UseField, SuperSelectField, useFormData } from '../../../../../../../../shared_imports'; +import { PhaseWithAllocation } from '../../../../../../../../../common/types'; + +import { DataTierAllocationType } from '../../../../../types'; + +import { NodeAllocation } from './node_allocation'; +import { SharedProps } from './types'; + +import './data_tier_allocation.scss'; + +type SelectOptions = EuiSuperSelectOption; + +const i18nTexts = { + allocationOptions: { + warm: { + default: { + input: i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.common.dataTierAllocation.warm.defaultOption.input', + { defaultMessage: 'Use warm nodes (recommended)' } + ), + helpText: i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.common.dataTierAllocation.warm.defaultOption.helpText', + { defaultMessage: 'Move data to nodes in the warm tier.' } + ), + }, + none: { + inputDisplay: i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.common.dataTierAllocation.warm.noneOption.input', + { defaultMessage: 'Off' } + ), + helpText: i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.common.dataTierAllocation.warm.noneOption.helpText', + { defaultMessage: 'Do not move data in the warm phase.' } + ), + }, + custom: { + inputDisplay: i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.common.dataTierAllocation.warm.customOption.input', + { defaultMessage: 'Custom' } + ), + helpText: i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.common.dataTierAllocation.warm.customOption.helpText', + { defaultMessage: 'Move data based on node attributes.' } + ), + }, + }, + cold: { + default: { + input: i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.common.dataTierAllocation.cold.defaultOption.input', + { defaultMessage: 'Use cold nodes (recommended)' } + ), + helpText: i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.common.dataTierAllocation.cold.defaultOption.helpText', + { defaultMessage: 'Move data to nodes in the cold tier.' } + ), + }, + none: { + inputDisplay: i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.common.dataTierAllocation.cold.noneOption.input', + { defaultMessage: 'Off' } + ), + helpText: i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.common.dataTierAllocation.cold.noneOption.helpText', + { defaultMessage: 'Do not move data in the cold phase.' } + ), + }, + custom: { + inputDisplay: i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.common.dataTierAllocation.cold.customOption.input', + { defaultMessage: 'Custom' } + ), + helpText: i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.common.dataTierAllocation.cold.customOption.helpText', + { defaultMessage: 'Move data based on node attributes.' } + ), + }, + }, + }, +}; + +const getSelectOptions = (phase: PhaseWithAllocation, disableDataTierOption: boolean) => + [ + disableDataTierOption + ? undefined + : { + 'data-test-subj': 'defaultDataAllocationOption', + value: 'node_roles', + inputDisplay: i18nTexts.allocationOptions[phase].default.input, + dropdownDisplay: ( + <> + {i18nTexts.allocationOptions[phase].default.input} + +

+ {i18nTexts.allocationOptions[phase].default.helpText} +

+ + + ), + }, + { + 'data-test-subj': 'customDataAllocationOption', + value: 'node_attrs', + inputDisplay: i18nTexts.allocationOptions[phase].custom.inputDisplay, + dropdownDisplay: ( + <> + {i18nTexts.allocationOptions[phase].custom.inputDisplay} + +

+ {i18nTexts.allocationOptions[phase].custom.helpText} +

+ + + ), + }, + { + 'data-test-subj': 'noneDataAllocationOption', + value: 'none', + inputDisplay: i18nTexts.allocationOptions[phase].none.inputDisplay, + dropdownDisplay: ( + <> + {i18nTexts.allocationOptions[phase].none.inputDisplay} + +

+ {i18nTexts.allocationOptions[phase].none.helpText} +

+ + + ), + }, + ].filter(Boolean) as SelectOptions[]; + +export const DataTierAllocation: FunctionComponent = (props) => { + const { phase, hasNodeAttributes, disableDataTierOption } = props; + + const dataTierAllocationTypePath = `_meta.${phase}.dataTierAllocationType`; + + const [formData] = useFormData({ + watch: dataTierAllocationTypePath, + }); + + const dataTierAllocationType = get(formData, dataTierAllocationTypePath); + + return ( +
+ + {(field) => { + /** + * We reset the value to "custom" if we deserialized to "default". + * + * It would be better if we had all the information we needed before deserializing and + * were able to handle this at the deserialization step instead of patching further down + * the component tree - this should be a future refactor. + */ + if (disableDataTierOption && field.value === 'node_roles') { + field.setValue('node_attrs'); + } + return ( + + ); + }} + + {dataTierAllocationType === 'node_attrs' && hasNodeAttributes && ( + <> + +
+ +
+ + )} +
+ ); +}; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/components/default_allocation_notice.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/components/default_allocation_notice.tsx new file mode 100644 index 00000000000000..f1f8fef62f1f87 --- /dev/null +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/components/default_allocation_notice.tsx @@ -0,0 +1,106 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { i18n } from '@kbn/i18n'; +import React, { FunctionComponent } from 'react'; +import { EuiCallOut } from '@elastic/eui'; + +import { PhaseWithAllocation, DataTierRole } from '../../../../../../../../../common/types'; + +import { AllocationNodeRole } from '../../../../../../../lib'; + +const i18nTextsNodeRoleToDataTier: Record = { + data_hot: i18n.translate('xpack.indexLifecycleMgmt.editPolicy.dataTierHotLabel', { + defaultMessage: 'hot', + }), + data_warm: i18n.translate('xpack.indexLifecycleMgmt.editPolicy.dataTierWarmLabel', { + defaultMessage: 'warm', + }), + data_cold: i18n.translate('xpack.indexLifecycleMgmt.editPolicy.dataTierColdLabel', { + defaultMessage: 'cold', + }), +}; + +const i18nTexts = { + notice: { + warm: { + title: i18n.translate( + 'xpack.indexLifecycleMgmt.warmPhase.dataTier.defaultAllocationNotice.warm.title', + { defaultMessage: 'No nodes assigned to the warm tier' } + ), + body: (nodeRole: DataTierRole) => + i18n.translate('xpack.indexLifecycleMgmt.warmPhase.dataTier.defaultAllocationNotice.warm', { + defaultMessage: + 'This policy will move data in the warm phase to {tier} tier nodes instead.', + values: { tier: i18nTextsNodeRoleToDataTier[nodeRole] }, + }), + }, + cold: { + title: i18n.translate( + 'xpack.indexLifecycleMgmt.warmPhase.dataTier.defaultAllocationNotice.cold.title', + { defaultMessage: 'No nodes assigned to the cold tier' } + ), + body: (nodeRole: DataTierRole) => + i18n.translate('xpack.indexLifecycleMgmt.warmPhase.dataTier.defaultAllocationNotice.cold', { + defaultMessage: + 'This policy will move data in the cold phase to {tier} tier nodes instead.', + values: { tier: i18nTextsNodeRoleToDataTier[nodeRole] }, + }), + }, + }, + warning: { + warm: { + title: i18n.translate( + 'xpack.indexLifecycleMgmt.warmPhase.dataTier.defaultAllocationNotAvailableTitle', + { defaultMessage: 'No nodes assigned to the warm tier' } + ), + body: i18n.translate( + 'xpack.indexLifecycleMgmt.warmPhase.dataTier.defaultAllocationNotAvailableBody', + { + defaultMessage: + 'Assign at least one node to the warm or hot tier to use role-based allocation. The policy will fail to complete allocation if there are no available nodes.', + } + ), + }, + cold: { + title: i18n.translate( + 'xpack.indexLifecycleMgmt.coldPhase.dataTier.defaultAllocationNotAvailableTitle', + { defaultMessage: 'No nodes assigned to the cold tier' } + ), + body: i18n.translate( + 'xpack.indexLifecycleMgmt.coldPhase.dataTier.defaultAllocationNotAvailableBody', + { + defaultMessage: + 'Assign at least one node to the cold, warm, or hot tier to use role-based allocation. The policy will fail to complete allocation if there are no available nodes.', + } + ), + }, + }, +}; + +interface Props { + phase: PhaseWithAllocation; + targetNodeRole: AllocationNodeRole; +} + +export const DefaultAllocationNotice: FunctionComponent = ({ phase, targetNodeRole }) => { + const content = + targetNodeRole === 'none' ? ( + + {i18nTexts.warning[phase].body} + + ) : ( + + {i18nTexts.notice[phase].body(targetNodeRole)} + + ); + + return content; +}; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/components/index.ts b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/components/index.ts new file mode 100644 index 00000000000000..0782782e77fadc --- /dev/null +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/components/index.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export { NodesDataProvider } from './node_data_provider'; + +export { NodeAllocation } from './node_allocation'; + +export { NodeAttrsDetails } from './node_attrs_details'; + +export { DataTierAllocation } from './data_tier_allocation'; + +export { DefaultAllocationNotice } from './default_allocation_notice'; + +export { NoNodeAttributesWarning } from './no_node_attributes_warning'; + +export { CloudDataTierCallout } from './cloud_data_tier_callout'; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/components/no_node_attributes_warning.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/components/no_node_attributes_warning.tsx new file mode 100644 index 00000000000000..338e5367a1d0df --- /dev/null +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/components/no_node_attributes_warning.tsx @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { FunctionComponent } from 'react'; +import { EuiCallOut } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; + +import { PhaseWithAllocation } from '../../../../../../../../../common/types'; + +const i18nTexts = { + title: i18n.translate('xpack.indexLifecycleMgmt.editPolicy.nodeAttributesMissingLabel', { + defaultMessage: 'No custom node attributes configured', + }), + warm: { + body: i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.warm.nodeAttributesMissingDescription', + { + defaultMessage: + 'Define custom node attributes in elasticsearch.yml to use attribute-based allocation. Warm nodes will be used instead.', + } + ), + }, + cold: { + body: i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.cold.nodeAttributesMissingDescription', + { + defaultMessage: + 'Define custom node attributes in elasticsearch.yml to use attribute-based allocation. Cold nodes will be used instead.', + } + ), + }, +}; + +export const NoNodeAttributesWarning: FunctionComponent<{ phase: PhaseWithAllocation }> = ({ + phase, +}) => { + return ( + + {i18nTexts[phase].body} + + ); +}; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/components/node_allocation.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/components/node_allocation.tsx new file mode 100644 index 00000000000000..407bb9ea92e855 --- /dev/null +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/components/node_allocation.tsx @@ -0,0 +1,119 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { useState, FunctionComponent } from 'react'; +import { get } from 'lodash'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { i18n } from '@kbn/i18n'; +import { EuiButtonEmpty, EuiText, EuiSpacer } from '@elastic/eui'; + +import { PhaseWithAllocationAction } from '../../../../../../../../../common/types'; + +import { UseField, SelectField, useFormData } from '../../../../../../../../shared_imports'; + +import { propertyof } from '../../../../../../../services/policies/policy_validation'; + +import { LearnMoreLink } from '../../../../learn_more_link'; + +import { NodeAttrsDetails } from './node_attrs_details'; + +import { SharedProps } from './types'; + +const learnMoreLink = ( + + } + docPath="modules-cluster.html#cluster-shard-allocation-settings" + /> +); + +const i18nTexts = { + doNotModifyAllocationOption: i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.nodeAllocation.doNotModifyAllocationOption', + { defaultMessage: 'Do not modify allocation configuration' } + ), +}; + +export const NodeAllocation: FunctionComponent = ({ phase, nodes }) => { + const allocationNodeAttributePath = `_meta.${phase}.allocationNodeAttribute`; + + const [formData] = useFormData({ + watch: [allocationNodeAttributePath], + }); + + const selectedAllocationNodeAttribute = get(formData, allocationNodeAttributePath); + + const [selectedNodeAttrsForDetails, setSelectedNodeAttrsForDetails] = useState( + null + ); + + const nodeOptions = Object.keys(nodes).map((attrs) => ({ + text: `${attrs} (${nodes[attrs].length})`, + value: attrs, + })); + + nodeOptions.sort((a, b) => a.value.localeCompare(b.value)); + + // check that this string is a valid property + const nodeAttrsProperty = propertyof('selectedNodeAttrs'); + + return ( + <> + +

+ +

+ + + + {/* + TODO: this field component must be revisited to support setting multiple require values and to support + setting `include and exclude values on ILM policies. See https://github.com/elastic/kibana/issues/77344 + */} + setSelectedNodeAttrsForDetails(selectedAllocationNodeAttribute)} + > + + + ) : undefined, + euiFieldProps: { + 'data-test-subj': `${phase}-${nodeAttrsProperty}`, + options: [{ text: i18nTexts.doNotModifyAllocationOption, value: '' }].concat( + nodeOptions + ), + hasNoInitialSelection: false, + }, + }} + /> + {selectedNodeAttrsForDetails ? ( + setSelectedNodeAttrsForDetails(null)} + /> + ) : null} + + ); +}; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/components/node_attrs_details.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/components/node_attrs_details.tsx new file mode 100644 index 00000000000000..61de977b1cb121 --- /dev/null +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/components/node_attrs_details.tsx @@ -0,0 +1,106 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n/react'; + +import { + EuiFlyoutBody, + EuiFlyout, + EuiTitle, + EuiInMemoryTable, + EuiSpacer, + EuiPortal, + EuiLoadingContent, + EuiCallOut, + EuiButton, +} from '@elastic/eui'; + +import { useLoadNodeDetails } from '../../../../../../../services/api'; + +interface Props { + close: () => void; + selectedNodeAttrs: string; +} + +export const NodeAttrsDetails: React.FunctionComponent = ({ close, selectedNodeAttrs }) => { + const { data, isLoading, error, resendRequest } = useLoadNodeDetails(selectedNodeAttrs); + let content; + if (isLoading) { + content = ; + } else if (error) { + const { statusCode, message } = error; + content = ( + + } + color="danger" + > +

+ {message} ({statusCode}) +

+ + + + + ); + } else { + content = ( + + ); + } + return ( + + + + +

+ +

+ + + {content} + + + + ); +}; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/components/node_data_provider.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/components/node_data_provider.tsx new file mode 100644 index 00000000000000..bb01a42e378faa --- /dev/null +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/components/node_data_provider.tsx @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; +import { EuiButton, EuiCallOut, EuiLoadingSpinner, EuiSpacer } from '@elastic/eui'; +import { FormattedMessage } from '@kbn/i18n/react'; + +import { ListNodesRouteResponse } from '../../../../../../../../../common/types'; +import { useLoadNodes } from '../../../../../../../services/api'; + +interface Props { + children: (data: ListNodesRouteResponse) => JSX.Element; +} + +export const NodesDataProvider = ({ children }: Props): JSX.Element => { + const { isLoading, data, error, resendRequest } = useLoadNodes(); + + if (isLoading) { + return ( + <> + + + + ); + } + + const renderError = () => { + if (error) { + const { statusCode, message } = error; + return ( + <> + + } + color="danger" + > +

+ {message} ({statusCode}) +

+ + + + + + + + ); + } + return null; + }; + + return ( + <> + {renderError()} + {/* `data` will always be defined because we use an initial value when loading */} + {children(data!)} + + ); +}; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/components/types.ts b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/components/types.ts new file mode 100644 index 00000000000000..7bd620757a8ac5 --- /dev/null +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/components/types.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { + ListNodesRouteResponse, + PhaseWithAllocation, +} from '../../../../../../../../../common/types'; + +export interface SharedProps { + phase: PhaseWithAllocation; + nodes: ListNodesRouteResponse['nodesByAttributes']; + hasNodeAttributes: boolean; + /** + * When on Cloud we want to disable the data tier allocation option when we detect that we are not + * using node roles in our Node config yet. See {@link ListNodesRouteResponse} for information about how this is + * detected. + */ + disableDataTierOption: boolean; +} diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/data_tier_allocation_field.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/data_tier_allocation_field.tsx new file mode 100644 index 00000000000000..73814537ff2764 --- /dev/null +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/data_tier_allocation_field.tsx @@ -0,0 +1,137 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { get } from 'lodash'; +import React, { FunctionComponent } from 'react'; +import { i18n } from '@kbn/i18n'; + +import { EuiDescribedFormGroup, EuiFormRow, EuiSpacer } from '@elastic/eui'; + +import { useKibana, useFormData } from '../../../../../../../shared_imports'; + +import { PhaseWithAllocation } from '../../../../../../../../common/types'; + +import { getAvailableNodeRoleForPhase } from '../../../../../../lib/data_tiers'; + +import { isNodeRoleFirstPreference } from '../../../../../../lib'; + +import { DataTierAllocationType } from '../../../../types'; + +import { + DataTierAllocation, + DefaultAllocationNotice, + NoNodeAttributesWarning, + NodesDataProvider, + CloudDataTierCallout, +} from './components'; + +const i18nTexts = { + title: i18n.translate('xpack.indexLifecycleMgmt.common.dataTier.title', { + defaultMessage: 'Data allocation', + }), +}; + +interface Props { + phase: PhaseWithAllocation; + description: React.ReactNode; +} + +/** + * Top-level layout control for the data tier allocation field. + */ +export const DataTierAllocationField: FunctionComponent = ({ phase, description }) => { + const { + services: { cloud }, + } = useKibana(); + + const dataTierAllocationTypePath = `_meta.${phase}.dataTierAllocationType`; + const [formData] = useFormData({ watch: dataTierAllocationTypePath }); + const allocationType: DataTierAllocationType = get(formData, dataTierAllocationTypePath); + + return ( + + {({ nodesByRoles, nodesByAttributes, isUsingDeprecatedDataRoleConfig }) => { + const hasDataNodeRoles = Object.keys(nodesByRoles).some((nodeRole) => + // match any of the "data_" roles, including data_content. + nodeRole.trim().startsWith('data_') + ); + const hasNodeAttrs = Boolean(Object.keys(nodesByAttributes ?? {}).length); + const isCloudEnabled = cloud?.isCloudEnabled ?? false; + + const renderNotice = () => { + switch (allocationType) { + case 'node_roles': + if (isCloudEnabled && phase === 'cold') { + const isUsingNodeRolesAllocation = + !isUsingDeprecatedDataRoleConfig && hasDataNodeRoles; + const hasNoNodesWithNodeRole = !nodesByRoles.data_cold?.length; + + if (isUsingNodeRolesAllocation && hasNoNodesWithNodeRole) { + // Tell cloud users they can deploy nodes on cloud. + return ( + <> + + + + ); + } + } + + const allocationNodeRole = getAvailableNodeRoleForPhase(phase, nodesByRoles); + if ( + allocationNodeRole === 'none' || + !isNodeRoleFirstPreference(phase, allocationNodeRole) + ) { + return ( + <> + + + + ); + } + break; + case 'node_attrs': + if (!hasNodeAttrs) { + return ( + <> + + + + ); + } + break; + default: + return null; + } + }; + + return ( + {i18nTexts.title}} + description={description} + fullWidth + > + + <> + + + {/* Data tier related warnings and call-to-action notices */} + {renderNotice()} + + + + ); + }} + + ); +}; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/index.ts b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/index.ts new file mode 100644 index 00000000000000..f9e939058adb99 --- /dev/null +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export { DataTierAllocationField } from './data_tier_allocation_field'; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_legacy_field.tsx similarity index 85% rename from x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field.tsx rename to x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_legacy_field.tsx index df59efcbfd2990..d64df468620e65 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_field.tsx +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/data_tier_allocation_legacy_field.tsx @@ -11,8 +11,7 @@ import { EuiDescribedFormGroup, EuiFormRow, EuiSpacer } from '@elastic/eui'; import { useKibana } from '../../../../../../shared_imports'; import { PhaseWithAllocationAction, PhaseWithAllocation } from '../../../../../../../common/types'; import { PhaseValidationErrors } from '../../../../../services/policies/policy_validation'; -import { getAvailableNodeRoleForPhase } from '../../../../../lib/data_tiers'; -import { isNodeRoleFirstPreference } from '../../../../../lib/data_tiers/is_node_role_first_preference'; +import { getAvailableNodeRoleForPhase, isNodeRoleFirstPreference } from '../../../../../lib'; import { DataTierAllocation, @@ -40,7 +39,7 @@ interface Props { /** * Top-level layout control for the data tier allocation field. */ -export const DataTierAllocationField: FunctionComponent = ({ +export const DataTierAllocationFieldLegacy: FunctionComponent = ({ description, phase, phaseData, @@ -55,6 +54,10 @@ export const DataTierAllocationField: FunctionComponent = ({ return ( {({ nodesByRoles, nodesByAttributes, isUsingDeprecatedDataRoleConfig }) => { + const hasDataNodeRoles = Object.keys(nodesByRoles).some((nodeRole) => + // match any of the "data_" roles, including data_content. + nodeRole.trim().startsWith('data_') + ); const hasNodeAttrs = Boolean(Object.keys(nodesByAttributes ?? {}).length); const renderNotice = () => { @@ -62,7 +65,8 @@ export const DataTierAllocationField: FunctionComponent = ({ case 'default': const isCloudEnabled = cloud?.isCloudEnabled ?? false; if (isCloudEnabled && phase === 'cold') { - const isUsingNodeRolesAllocation = !isUsingDeprecatedDataRoleConfig; + const isUsingNodeRolesAllocation = + !isUsingDeprecatedDataRoleConfig && hasDataNodeRoles; const hasNoNodesWithNodeRole = !nodesByRoles.data_cold?.length; if (isUsingNodeRolesAllocation && hasNoNodesWithNodeRole) { @@ -120,9 +124,9 @@ export const DataTierAllocationField: FunctionComponent = ({ phaseData={phaseData} isShowingErrors={isShowingErrors} nodes={nodesByAttributes} - disableDataTierOption={ - !!(isUsingDeprecatedDataRoleConfig && cloud?.isCloudEnabled) - } + disableDataTierOption={Boolean( + cloud?.isCloudEnabled && !hasDataNodeRoles && isUsingDeprecatedDataRoleConfig + )} /> {/* Data tier related warnings and call-to-action notices */} diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/forcemerge_field.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/forcemerge_field.tsx index c6f02fd2191301..b410bd0e6b3b03 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/forcemerge_field.tsx +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/forcemerge_field.tsx @@ -3,33 +3,32 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { get } from 'lodash'; -import { FormattedMessage } from '@kbn/i18n/react'; -import { EuiDescribedFormGroup, EuiSpacer, EuiTextColor } from '@elastic/eui'; -import React from 'react'; -import { Phases } from '../../../../../../../common/types'; +import React, { useMemo } from 'react'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { EuiSpacer, EuiTextColor } from '@elastic/eui'; -import { UseField, ToggleField, NumericField, useFormData } from '../../../../../../shared_imports'; +import { UseField, ToggleField, NumericField } from '../../../../../../shared_imports'; import { i18nTexts } from '../../../i18n_texts'; -import { LearnMoreLink } from '../../'; +import { useEditPolicyContext } from '../../../edit_policy_context'; + +import { LearnMoreLink, DescribedFormField } from '../../'; interface Props { - phase: keyof Phases & string; + phase: 'hot' | 'warm'; } -const forceMergeEnabledPath = '_meta.hot.forceMergeEnabled'; - export const Forcemerge: React.FunctionComponent = ({ phase }) => { - const [formData] = useFormData({ - watch: forceMergeEnabledPath, - }); - const forceMergeEnabled = get(formData, forceMergeEnabledPath); + const { originalPolicy } = useEditPolicyContext(); + + const initialToggleValue = useMemo(() => { + return Boolean(originalPolicy.phases[phase]?.actions?.forcemerge); + }, [originalPolicy, phase]); return ( - = ({ phase }) => { } titleSize="xs" fullWidth + switchProps={{ + 'aria-label': i18nTexts.editPolicy.forceMergeEnabledFieldLabel, + 'data-test-subj': `${phase}-forceMergeSwitch`, + 'aria-controls': 'forcemergeContent', + label: i18nTexts.editPolicy.forceMergeEnabledFieldLabel, + initialValue: initialToggleValue, + }} > -
- {forceMergeEnabled && ( - <> - - - - )} + +
- + ); }; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/index.ts b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/index.ts index 3b94d36a977d14..0cae3eea6316b9 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/index.ts +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/index.ts @@ -6,8 +6,12 @@ export { useRolloverPath } from '../../../constants'; +export { DataTierAllocationFieldLegacy } from './data_tier_allocation_legacy_field'; + export { DataTierAllocationField } from './data_tier_allocation_field'; export { Forcemerge } from './forcemerge_field'; export { SetPriorityInput } from './set_priority_input'; + +export { MinAgeInputField } from './min_age_input_field'; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/min_age_input_field/index.ts b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/min_age_input_field/index.ts new file mode 100644 index 00000000000000..0228a524f8129c --- /dev/null +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/min_age_input_field/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export { MinAgeInputField } from './min_age_input_field'; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/min_age_input_field/min_age_input_field.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/min_age_input_field/min_age_input_field.tsx new file mode 100644 index 00000000000000..f37c3873544182 --- /dev/null +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/min_age_input_field/min_age_input_field.tsx @@ -0,0 +1,212 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import React, { FunctionComponent } from 'react'; +import { get } from 'lodash'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { i18n } from '@kbn/i18n'; + +import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; + +import { + useFormData, + UseField, + NumericField, + SelectField, +} from '../../../../../../../shared_imports'; + +import { LearnMoreLink } from '../../../learn_more_link'; +import { useRolloverPath } from '../../../../constants'; + +import { getUnitsAriaLabelForPhase, getTimingLabelForPhase } from './util'; + +type PhaseWithMinAgeAction = 'warm' | 'cold' | 'delete'; + +interface Props { + phase: PhaseWithMinAgeAction; +} + +export const MinAgeInputField: FunctionComponent = ({ phase }): React.ReactElement => { + const [formData] = useFormData({ watch: useRolloverPath }); + const rolloverEnabled = get(formData, useRolloverPath); + + let daysOptionLabel; + let hoursOptionLabel; + let minutesOptionLabel; + let secondsOptionLabel; + let millisecondsOptionLabel; + let microsecondsOptionLabel; + let nanosecondsOptionLabel; + + if (rolloverEnabled) { + daysOptionLabel = i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.rolloverDaysOptionLabel', + { + defaultMessage: 'days from rollover', + } + ); + + hoursOptionLabel = i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.rolloverHoursOptionLabel', + { + defaultMessage: 'hours from rollover', + } + ); + minutesOptionLabel = i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.rolloverMinutesOptionLabel', + { + defaultMessage: 'minutes from rollover', + } + ); + + secondsOptionLabel = i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.rolloverSecondsOptionLabel', + { + defaultMessage: 'seconds from rollover', + } + ); + millisecondsOptionLabel = i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.rolloverMilliSecondsOptionLabel', + { + defaultMessage: 'milliseconds from rollover', + } + ); + + microsecondsOptionLabel = i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.rolloverMicroSecondsOptionLabel', + { + defaultMessage: 'microseconds from rollover', + } + ); + + nanosecondsOptionLabel = i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.rolloverNanoSecondsOptionLabel', + { + defaultMessage: 'nanoseconds from rollover', + } + ); + } else { + daysOptionLabel = i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.creationDaysOptionLabel', + { + defaultMessage: 'days from index creation', + } + ); + + hoursOptionLabel = i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.creationHoursOptionLabel', + { + defaultMessage: 'hours from index creation', + } + ); + + minutesOptionLabel = i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.creationMinutesOptionLabel', + { + defaultMessage: 'minutes from index creation', + } + ); + + secondsOptionLabel = i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.creationSecondsOptionLabel', + { + defaultMessage: 'seconds from index creation', + } + ); + + millisecondsOptionLabel = i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.creationMilliSecondsOptionLabel', + { + defaultMessage: 'milliseconds from index creation', + } + ); + + microsecondsOptionLabel = i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.creationMicroSecondsOptionLabel', + { + defaultMessage: 'microseconds from index creation', + } + ); + + nanosecondsOptionLabel = i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.creationNanoSecondsOptionLabel', + { + defaultMessage: 'nanoseconds from index creation', + } + ); + } + + return ( + + + + } + /> + ), + euiFieldProps: { + 'data-test-subj': `${phase}-selectedMinimumAge`, + min: 0, + }, + }} + /> + + + + + + ); +}; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/min_age_input_field/util.ts b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/min_age_input_field/util.ts new file mode 100644 index 00000000000000..181894badba7bf --- /dev/null +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/shared/min_age_input_field/util.ts @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { i18n } from '@kbn/i18n'; +import { Phases } from '../../../../../../../../common/types'; + +type PhaseWithMinAgeAction = 'warm' | 'cold' | 'delete'; + +export function getUnitsAriaLabelForPhase(phase: keyof Phases) { + // NOTE: Hot phase isn't necessary, because indices begin in the hot phase. + switch (phase) { + case 'warm': + return i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.phaseWarm.minimumAgeUnitsAriaLabel', + { + defaultMessage: 'Units for timing of warm phase', + } + ); + + case 'cold': + return i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.phaseCold.minimumAgeUnitsAriaLabel', + { + defaultMessage: 'Units for timing of cold phase', + } + ); + + case 'delete': + return i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.phaseDelete.minimumAgeUnitsAriaLabel', + { + defaultMessage: 'Units for timing of delete phase', + } + ); + } +} +export function getTimingLabelForPhase(phase: PhaseWithMinAgeAction) { + // NOTE: Hot phase isn't necessary, because indices begin in the hot phase. + switch (phase) { + case 'warm': + return i18n.translate('xpack.indexLifecycleMgmt.editPolicy.phaseWarm.minimumAgeLabel', { + defaultMessage: 'Timing for warm phase', + }); + + case 'cold': + return i18n.translate('xpack.indexLifecycleMgmt.editPolicy.phaseCold.minimumAgeLabel', { + defaultMessage: 'Timing for cold phase', + }); + + case 'delete': + return i18n.translate('xpack.indexLifecycleMgmt.editPolicy.phaseDelete.minimumAgeLabel', { + defaultMessage: 'Timing for delete phase', + }); + } +} diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/warm_phase.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/warm_phase.tsx deleted file mode 100644 index d0aaec367104e4..00000000000000 --- a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/warm_phase.tsx +++ /dev/null @@ -1,316 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import React, { Fragment, FunctionComponent } from 'react'; -import { get } from 'lodash'; -import { FormattedMessage } from '@kbn/i18n/react'; -import { i18n } from '@kbn/i18n'; -import { - EuiTextColor, - EuiFlexGroup, - EuiFlexItem, - EuiSpacer, - EuiFormRow, - EuiFieldNumber, - EuiSwitch, - EuiDescribedFormGroup, -} from '@elastic/eui'; - -import { useFormData } from '../../../../../shared_imports'; -import { Phases, WarmPhase as WarmPhaseInterface } from '../../../../../../common/types'; -import { PhaseValidationErrors } from '../../../../services/policies/policy_validation'; - -import { useRolloverPath } from './shared'; - -import { - LearnMoreLink, - ActiveBadge, - PhaseErrorMessage, - OptionalLabel, - ErrableFormRow, - SetPriorityInput, - MinAgeInput, - DescribedFormField, - Forcemerge, -} from '../'; - -import { DataTierAllocationField } from './shared'; - -const i18nTexts = { - shrinkLabel: i18n.translate('xpack.indexLifecycleMgmt.warmPhase.shrinkIndexLabel', { - defaultMessage: 'Shrink index', - }), - moveToWarmPhaseOnRolloverLabel: i18n.translate( - 'xpack.indexLifecycleMgmt.warmPhase.moveToWarmPhaseOnRolloverLabel', - { - defaultMessage: 'Move to warm phase on rollover', - } - ), - dataTierAllocation: { - description: i18n.translate('xpack.indexLifecycleMgmt.warmPhase.dataTier.description', { - defaultMessage: 'Move data to nodes optimized for less-frequent, read-only access.', - }), - }, -}; - -const warmProperty: keyof Phases = 'warm'; -const phaseProperty = (propertyName: keyof WarmPhaseInterface) => propertyName; - -interface Props { - setPhaseData: ( - key: keyof WarmPhaseInterface & string, - value: boolean | string | undefined - ) => void; - phaseData: WarmPhaseInterface; - isShowingErrors: boolean; - errors?: PhaseValidationErrors; -} -export const WarmPhase: FunctionComponent = ({ - setPhaseData, - phaseData, - errors, - isShowingErrors, -}) => { - const [formData] = useFormData({ - watch: useRolloverPath, - }); - - const hotPhaseRolloverEnabled = get(formData, useRolloverPath); - - return ( -
- <> - -

- -

{' '} - {phaseData.phaseEnabled && !isShowingErrors ? : null} - -
- } - titleSize="s" - description={ - -

- -

- - } - id={`${warmProperty}-${phaseProperty('phaseEnabled')}`} - checked={phaseData.phaseEnabled} - onChange={(e) => { - setPhaseData(phaseProperty('phaseEnabled'), e.target.checked); - }} - aria-controls="warmPhaseContent" - /> - - } - fullWidth - > - - {phaseData.phaseEnabled ? ( - - {hotPhaseRolloverEnabled ? ( - - { - setPhaseData(phaseProperty('warmPhaseOnRollover'), e.target.checked); - }} - /> - - ) : null} - {!phaseData.warmPhaseOnRollover || !hotPhaseRolloverEnabled ? ( - - - - errors={errors} - phaseData={phaseData} - phase={warmProperty} - isShowingErrors={isShowingErrors} - setPhaseData={setPhaseData} - rolloverEnabled={hotPhaseRolloverEnabled} - /> - - ) : null} - - ) : null} - - - - {phaseData.phaseEnabled ? ( - - {/* Data tier allocation section */} - - - - {i18n.translate('xpack.indexLifecycleMgmt.warmPhase.replicasTitle', { - defaultMessage: 'Replicas', - })} - - } - description={i18n.translate( - 'xpack.indexLifecycleMgmt.warmPhase.numberOfReplicasDescription', - { - defaultMessage: - 'Set the number of replicas. Remains the same as the previous phase by default.', - } - )} - switchProps={{ - label: i18n.translate( - 'xpack.indexLifecycleMgmt.editPolicy.warmPhase.numberOfReplicas.switchLabel', - { defaultMessage: 'Set replicas' } - ), - initialValue: Boolean(phaseData.selectedReplicaCount), - onChange: (v) => { - if (!v) { - setPhaseData('selectedReplicaCount', ''); - } - }, - }} - fullWidth - > - - - - - } - isShowingErrors={isShowingErrors} - errors={errors?.selectedReplicaCount} - > - { - setPhaseData('selectedReplicaCount', e.target.value); - }} - min={0} - /> - - - - - - } - description={ - - {' '} - - - } - fullWidth - titleSize="xs" - > - - { - setPhaseData(phaseProperty('shrinkEnabled'), e.target.checked); - }} - label={i18nTexts.shrinkLabel} - aria-label={i18nTexts.shrinkLabel} - aria-controls="shrinkContent" - /> - -
- {phaseData.shrinkEnabled ? ( - - - - - - { - setPhaseData( - phaseProperty('selectedPrimaryShardCount'), - e.target.value - ); - }} - min={1} - /> - - - - - - ) : null} -
- - - - - errors={errors} - phaseData={phaseData} - phase={warmProperty} - isShowingErrors={isShowingErrors} - setPhaseData={setPhaseData} - /> - - ) : null} - -
- ); -}; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/warm_phase/index.ts b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/warm_phase/index.ts new file mode 100644 index 00000000000000..d0686270ba559b --- /dev/null +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/warm_phase/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export { WarmPhase } from './warm_phase'; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/warm_phase/warm_phase.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/warm_phase/warm_phase.tsx new file mode 100644 index 00000000000000..7b1a4f44b5de63 --- /dev/null +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/phases/warm_phase/warm_phase.tsx @@ -0,0 +1,233 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { Fragment, FunctionComponent } from 'react'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { i18n } from '@kbn/i18n'; +import { get } from 'lodash'; + +import { + EuiTextColor, + EuiFlexGroup, + EuiFlexItem, + EuiSpacer, + EuiDescribedFormGroup, +} from '@elastic/eui'; + +import { + useFormData, + UseField, + ToggleField, + useFormContext, + NumericField, +} from '../../../../../../shared_imports'; + +import { Phases } from '../../../../../../../common/types'; + +import { useRolloverPath, MinAgeInputField, Forcemerge, SetPriorityInput } from '../shared'; + +import { useEditPolicyContext } from '../../../edit_policy_context'; + +import { LearnMoreLink, ActiveBadge, PhaseErrorMessage, DescribedFormField } from '../../'; + +import { DataTierAllocationField } from '../shared'; + +const i18nTexts = { + shrinkLabel: i18n.translate('xpack.indexLifecycleMgmt.warmPhase.shrinkIndexLabel', { + defaultMessage: 'Shrink index', + }), + dataTierAllocation: { + description: i18n.translate('xpack.indexLifecycleMgmt.warmPhase.dataTier.description', { + defaultMessage: 'Move data to nodes optimized for less-frequent, read-only access.', + }), + }, +}; + +const warmProperty: keyof Phases = 'warm'; + +export const WarmPhase: FunctionComponent = () => { + const { originalPolicy } = useEditPolicyContext(); + const form = useFormContext(); + const [formData] = useFormData({ + watch: [useRolloverPath, '_meta.warm.enabled', '_meta.warm.warmPhaseOnRollover'], + }); + + const enabled = get(formData, '_meta.warm.enabled'); + const hotPhaseRolloverEnabled = get(formData, useRolloverPath); + const warmPhaseOnRollover = get(formData, '_meta.warm.warmPhaseOnRollover'); + const isShowingErrors = form.isValid === false; + + return ( +
+ <> + +

+ +

{' '} + {enabled && !isShowingErrors ? : null} + +
+ } + titleSize="s" + description={ + +

+ +

+ + + } + fullWidth + > + + {enabled && ( + + {hotPhaseRolloverEnabled && ( + + )} + {(!warmPhaseOnRollover || !hotPhaseRolloverEnabled) && ( + <> + + + + )} + + )} + + + + {enabled && ( + + {/* Data tier allocation section */} + + + + {i18n.translate('xpack.indexLifecycleMgmt.warmPhase.replicasTitle', { + defaultMessage: 'Replicas', + })} + + } + description={i18n.translate( + 'xpack.indexLifecycleMgmt.warmPhase.numberOfReplicasDescription', + { + defaultMessage: + 'Set the number of replicas. Remains the same as the previous phase by default.', + } + )} + switchProps={{ + 'data-test-subj': 'warm-setReplicasSwitch', + label: i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.warmPhase.numberOfReplicas.switchLabel', + { defaultMessage: 'Set replicas' } + ), + initialValue: Boolean( + originalPolicy.phases.warm?.actions?.allocate?.number_of_replicas + ), + }} + fullWidth + > + + + + + + } + description={ + + {' '} + + + } + titleSize="xs" + switchProps={{ + 'aria-controls': 'shrinkContent', + 'data-test-subj': 'shrinkSwitch', + label: i18nTexts.shrinkLabel, + 'aria-label': i18nTexts.shrinkLabel, + initialValue: Boolean(originalPolicy.phases.warm?.actions?.shrink), + }} + fullWidth + > +
+ + + + + + + +
+ + + + + + + )} + + + ); +}; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/policy_json_flyout.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/policy_json_flyout.tsx index e9ce193118b35b..9ed24355ce7b36 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/policy_json_flyout.tsx +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/policy_json_flyout.tsx @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import React, { useEffect, useState } from 'react'; +import React, { useCallback, useEffect, useState } from 'react'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; @@ -25,6 +25,7 @@ import { import { SerializedPolicy } from '../../../../../common/types'; import { useFormContext, useFormData } from '../../../../shared_imports'; +import { FormInternal } from '../types'; interface Props { legacyPolicy: SerializedPolicy; @@ -44,27 +45,29 @@ export const PolicyJsonFlyout: React.FunctionComponent = ({ */ const [policy, setPolicy] = useState(undefined); - const form = useFormContext(); - const [formData, getFormData] = useFormData(); + const { validate: validateForm } = useFormContext(); + const [, getFormData] = useFormData(); + + const updatePolicy = useCallback(async () => { + setPolicy(undefined); + if (await validateForm()) { + const p = getFormData() as SerializedPolicy; + setPolicy({ + ...legacyPolicy, + phases: { + ...legacyPolicy.phases, + hot: p.phases.hot, + warm: p.phases.warm, + }, + }); + } else { + setPolicy(null); + } + }, [setPolicy, getFormData, legacyPolicy, validateForm]); useEffect(() => { - (async function checkPolicy() { - setPolicy(undefined); - if (await form.validate()) { - const p = getFormData() as SerializedPolicy; - setPolicy({ - ...legacyPolicy, - phases: { - ...legacyPolicy.phases, - hot: p.phases.hot, - }, - }); - } else { - setPolicy(null); - } - })(); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [form, legacyPolicy, formData]); + updatePolicy(); + }, [updatePolicy]); let content: React.ReactNode; switch (policy) { diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/set_priority_input.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/set_priority_input_legacy.tsx similarity index 100% rename from x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/set_priority_input.tsx rename to x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/set_priority_input_legacy.tsx diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/toggleable_field.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/toggleable_field.tsx index ff4301808db339..d188a172d746be 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/toggleable_field.tsx +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/components/toggleable_field.tsx @@ -9,7 +9,7 @@ import { EuiSpacer, EuiSwitch, EuiSwitchProps } from '@elastic/eui'; export interface Props extends Omit { initialValue: boolean; - onChange: (nextValue: boolean) => void; + onChange?: (nextValue: boolean) => void; } export const ToggleableField: FunctionComponent = ({ @@ -28,7 +28,9 @@ export const ToggleableField: FunctionComponent = ({ onChange={(e) => { const nextValue = e.target.checked; setIsContentVisible(nextValue); - onChange(nextValue); + if (onChange) { + onChange(nextValue); + } }} /> diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/deserializer.ts b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/deserializer.ts index bb24eea64ec8cc..760c6ad713ea04 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/deserializer.ts +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/deserializer.ts @@ -10,22 +10,32 @@ import { SerializedPolicy } from '../../../../common/types'; import { splitSizeAndUnits } from '../../services/policies/policy_serialization'; +import { determineDataTierAllocationType } from '../../lib'; + import { FormInternal } from './types'; -export const deserializer = (policy: SerializedPolicy): FormInternal => - produce( +export const deserializer = (policy: SerializedPolicy): FormInternal => { + const _meta: FormInternal['_meta'] = { + hot: { + useRollover: Boolean(policy.phases.hot?.actions?.rollover), + forceMergeEnabled: Boolean(policy.phases.hot?.actions?.forcemerge), + bestCompression: policy.phases.hot?.actions?.forcemerge?.index_codec === 'best_compression', + }, + warm: { + enabled: Boolean(policy.phases.warm), + warmPhaseOnRollover: Boolean(policy.phases.warm?.min_age === '0ms'), + forceMergeEnabled: Boolean(policy.phases.warm?.actions?.forcemerge), + bestCompression: policy.phases.warm?.actions?.forcemerge?.index_codec === 'best_compression', + dataTierAllocationType: determineDataTierAllocationType(policy.phases.warm?.actions), + }, + }; + + return produce( { ...policy, - _meta: { - hot: { - useRollover: Boolean(policy.phases.hot?.actions?.rollover), - forceMergeEnabled: Boolean(policy.phases.hot?.actions?.forcemerge), - bestCompression: - policy.phases.hot?.actions?.forcemerge?.index_codec === 'best_compression', - }, - }, + _meta, }, - (draft) => { + (draft: FormInternal) => { if (draft.phases.hot?.actions?.rollover) { if (draft.phases.hot.actions.rollover.max_size) { const maxSize = splitSizeAndUnits(draft.phases.hot.actions.rollover.max_size); @@ -39,5 +49,20 @@ export const deserializer = (policy: SerializedPolicy): FormInternal => draft._meta.hot.maxAgeUnit = maxAge.units; } } + + if (draft.phases.warm) { + if (draft.phases.warm.actions?.allocate?.require) { + Object.entries(draft.phases.warm.actions.allocate.require).forEach((entry) => { + draft._meta.warm.allocationNodeAttribute = entry.join(':'); + }); + } + + if (draft.phases.warm.min_age) { + const minAge = splitSizeAndUnits(draft.phases.warm.min_age); + draft.phases.warm.min_age = minAge.size; + draft._meta.warm.minAgeUnit = minAge.units; + } + } } ); +}; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/edit_policy.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/edit_policy.tsx index 8f8b0447f378a4..eecdfb4871a676 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/edit_policy.tsx +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/edit_policy.tsx @@ -67,6 +67,8 @@ import { schema } from './form_schema'; import { deserializer } from './deserializer'; import { createSerializer } from './serializer'; +import { EditPolicyContextProvider } from './edit_policy_context'; + export interface Props { policies: PolicyFromES[]; policyName: string; @@ -89,6 +91,7 @@ const mergeAllSerializedPolicies = ( phases: { ...legacySerializedPolicy.phases, hot: serializedPolicy.phases.hot, + warm: serializedPolicy.phases.warm, }, }; }; @@ -113,9 +116,11 @@ export const EditPolicy: React.FunctionComponent = ({ return createSerializer(existingPolicy?.policy); }, [existingPolicy?.policy]); + const originalPolicy = existingPolicy?.policy ?? defaultPolicy; + const { form } = useForm({ schema, - defaultValue: existingPolicy?.policy ?? defaultPolicy, + defaultValue: originalPolicy, deserializer, serializer, }); @@ -132,22 +137,6 @@ export const EditPolicy: React.FunctionComponent = ({ history.push('/policies'); }; - const setWarmPhaseOnRollover = useCallback( - (value: boolean) => { - setPolicy((p) => ({ - ...p, - phases: { - ...p.phases, - warm: { - ...p.phases.warm, - warmPhaseOnRollover: value, - }, - }, - })); - }, - [setPolicy] - ); - const submit = async () => { setIsShowingErrors(true); const { data: formLibPolicy, isValid: newIsValid } = await form.submit(); @@ -206,10 +195,6 @@ export const EditPolicy: React.FunctionComponent = ({ [setPolicy] ); - const setWarmPhaseData = useCallback( - (key: string, value: any) => setPhaseData('warm', key, value), - [setPhaseData] - ); const setColdPhaseData = useCallback( (key: string, value: any) => setPhaseData('cold', key, value), [setPhaseData] @@ -220,234 +205,236 @@ export const EditPolicy: React.FunctionComponent = ({ ); return ( - - - - -

- {isNewPolicy - ? i18n.translate('xpack.indexLifecycleMgmt.editPolicy.createPolicyMessage', { - defaultMessage: 'Create an index lifecycle policy', - }) - : i18n.translate('xpack.indexLifecycleMgmt.editPolicy.editPolicyMessage', { - defaultMessage: 'Edit index lifecycle policy {originalPolicyName}', - values: { originalPolicyName }, - })} -

- - -
-
- - -

- + +

+ {isNewPolicy + ? i18n.translate('xpack.indexLifecycleMgmt.editPolicy.createPolicyMessage', { + defaultMessage: 'Create an index lifecycle policy', + }) + : i18n.translate('xpack.indexLifecycleMgmt.editPolicy.editPolicyMessage', { + defaultMessage: 'Edit index lifecycle policy {originalPolicyName}', + values: { originalPolicyName }, + })} +

+ + +
+ + + +

+ {' '} - - } - /> -

- + />{' '} + + } + /> +

+ - + - {isNewPolicy ? null : ( - - -

- + {isNewPolicy ? null : ( + + +

+ + + + .{' '} - - .{' '} - +

+ + + + + { + setSaveAsNew(e.target.checked); + }} + label={ + + + + } /> -

- - - - - { - setSaveAsNew(e.target.checked); - }} - label={ - + + + )} + + {saveAsNew || isNewPolicy ? ( + + - } - /> - - - )} - - {saveAsNew || isNewPolicy ? ( - - +
+ } + titleSize="s" + fullWidth + > + - -
- } - titleSize="s" - fullWidth - > - + { + setPolicy({ ...policy, name: e.target.value }); + }} /> - } - > - { - setPolicy({ ...policy, name: e.target.value }); - }} - /> - - - ) : null} - - - - - - - - 0} - setPhaseData={setWarmPhaseData} - phaseData={policy.phases.warm} - /> - - - - 0} - setPhaseData={setColdPhaseData} - phaseData={policy.phases.cold} - /> - - - - 0 - } - getUrlForApp={getUrlForApp} - setPhaseData={setDeletePhaseData} - phaseData={policy.phases.delete} - /> - - - - - - - - - {saveAsNew ? ( - - ) : ( + + + ) : null} + + + + + + + + + + + + 0 + } + setPhaseData={setColdPhaseData} + phaseData={policy.phases.cold} + /> + + + + 0 + } + getUrlForApp={getUrlForApp} + setPhaseData={setDeletePhaseData} + phaseData={policy.phases.delete} + /> + + + + + + + + + {saveAsNew ? ( + + ) : ( + + )} + + + + + - )} - - - - - + + + + + + + + {isShowingPolicyJsonFlyout ? ( - - - - - - - - {isShowingPolicyJsonFlyout ? ( - - ) : ( - - )} - - - - - {isShowingPolicyJsonFlyout ? ( - setIsShowingPolicyJsonFlyout(false)} - /> - ) : null} - - - - - + ) : ( + + )} + + + + + {isShowingPolicyJsonFlyout ? ( + setIsShowingPolicyJsonFlyout(false)} + /> + ) : null} + + + + + + ); }; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/edit_policy_context.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/edit_policy_context.tsx new file mode 100644 index 00000000000000..4748c26d6cec1b --- /dev/null +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/edit_policy_context.tsx @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { createContext, ReactChild, useContext } from 'react'; +import { SerializedPolicy } from '../../../../common/types'; + +interface EditPolicyContextValue { + originalPolicy: SerializedPolicy; +} + +const EditPolicyContext = createContext(null as any); + +export const EditPolicyContextProvider = ({ + value, + children, +}: { + value: EditPolicyContextValue; + children: ReactChild; +}) => { + return {children}; +}; + +export const useEditPolicyContext = () => { + const ctx = useContext(EditPolicyContext); + if (!ctx) { + throw new Error('useEditPolicyContext can only be called inside of EditPolicyContext!'); + } + return ctx; +}; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/form_schema.ts b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/form_schema.ts index 806164c8b0da12..a80382e87539c2 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/form_schema.ts +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/form_schema.ts @@ -7,13 +7,19 @@ import { i18n } from '@kbn/i18n'; import { FormSchema, fieldValidators } from '../../../shared_imports'; -import { defaultSetPriority } from '../../constants'; +import { defaultSetPriority, defaultPhaseIndexPriority } from '../../constants'; import { FormInternal } from './types'; + import { ifExistsNumberGreaterThanZero, rolloverThresholdsValidator } from './form_validations'; + import { i18nTexts } from './i18n_texts'; -const { emptyField } = fieldValidators; +const { emptyField, numberGreaterThanField } = fieldValidators; + +const serializers = { + stringToNumber: (v: string): any => (v ? parseInt(v, 10) : undefined), +}; export const schema: FormSchema = { _meta: { @@ -30,21 +36,38 @@ export const schema: FormSchema = { maxAgeUnit: { defaultValue: 'd', }, - forceMergeEnabled: { - label: i18nTexts.editPolicy.forceMergeEnabledFieldLabel, - }, bestCompression: { - label: i18n.translate('xpack.indexLifecycleMgmt.forcemerge.bestCompressionLabel', { - defaultMessage: 'Compress stored fields', - }), - helpText: i18n.translate( - 'xpack.indexLifecycleMgmt.editPolicy.forceMerge.bestCompressionText', - { - defaultMessage: - 'Use higher compression for stored fields at the cost of slower performance.', - } + label: i18nTexts.editPolicy.bestCompressionFieldLabel, + helpText: i18nTexts.editPolicy.bestCompressionFieldHelpText, + }, + }, + warm: { + enabled: { + defaultValue: false, + label: i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.warmPhase.activateWarmPhaseSwitchLabel', + { defaultMessage: 'Activate warm phase' } ), }, + warmPhaseOnRollover: { + defaultValue: true, + label: i18n.translate('xpack.indexLifecycleMgmt.warmPhase.moveToWarmPhaseOnRolloverLabel', { + defaultMessage: 'Move to warm phase on rollover', + }), + }, + minAgeUnit: { + defaultValue: 'ms', + }, + bestCompression: { + label: i18nTexts.editPolicy.bestCompressionFieldLabel, + helpText: i18nTexts.editPolicy.bestCompressionFieldHelpText, + }, + dataTierAllocationType: { + label: i18nTexts.editPolicy.allocationTypeOptionsFieldLabel, + }, + allocationNodeAttribute: { + label: i18nTexts.editPolicy.allocationNodeAttributeFieldLabel, + }, }, }, phases: { @@ -76,7 +99,7 @@ export const schema: FormSchema = { validator: ifExistsNumberGreaterThanZero, }, ], - serializer: (v: string): any => (v ? parseInt(v, 10) : undefined), + serializer: serializers.stringToNumber, }, max_size: { label: i18n.translate('xpack.indexLifecycleMgmt.hotPhase.maximumIndexSizeLabel', { @@ -94,9 +117,7 @@ export const schema: FormSchema = { }, forcemerge: { max_num_segments: { - label: i18n.translate('xpack.indexLifecycleMgmt.forceMerge.numberOfSegmentsLabel', { - defaultMessage: 'Number of segments', - }), + label: i18nTexts.editPolicy.maxNumSegmentsFieldLabel, validations: [ { validator: emptyField( @@ -110,17 +131,94 @@ export const schema: FormSchema = { validator: ifExistsNumberGreaterThanZero, }, ], - serializer: (v: string): any => (v ? parseInt(v, 10) : undefined), + serializer: serializers.stringToNumber, }, }, set_priority: { priority: { defaultValue: defaultSetPriority as any, - label: i18n.translate('xpack.indexLifecycleMgmt.editPolicy.indexPriorityLabel', { - defaultMessage: 'Index priority (optional)', + label: i18nTexts.editPolicy.setPriorityFieldLabel, + validations: [{ validator: ifExistsNumberGreaterThanZero }], + serializer: serializers.stringToNumber, + }, + }, + }, + }, + warm: { + min_age: { + defaultValue: '0', + validations: [ + { + validator: (arg) => + numberGreaterThanField({ + than: 0, + allowEquality: true, + message: i18nTexts.editPolicy.errors.nonNegativeNumberRequired, + })({ + ...arg, + value: arg.value === '' ? -Infinity : parseInt(arg.value, 10), + }), + }, + ], + }, + actions: { + allocate: { + number_of_replicas: { + label: i18n.translate('xpack.indexLifecycleMgmt.warmPhase.numberOfReplicasLabel', { + defaultMessage: 'Number of replicas (optional)', + }), + validations: [ + { + validator: ifExistsNumberGreaterThanZero, + }, + ], + serializer: serializers.stringToNumber, + }, + }, + shrink: { + number_of_shards: { + label: i18n.translate('xpack.indexLifecycleMgmt.warmPhase.numberOfPrimaryShardsLabel', { + defaultMessage: 'Number of primary shards', }), + validations: [ + { + validator: emptyField(i18nTexts.editPolicy.errors.numberRequired), + }, + { + validator: numberGreaterThanField({ + message: i18nTexts.editPolicy.errors.numberGreatThan0Required, + than: 0, + }), + }, + ], + serializer: serializers.stringToNumber, + }, + }, + forcemerge: { + max_num_segments: { + label: i18nTexts.editPolicy.maxNumSegmentsFieldLabel, + validations: [ + { + validator: emptyField( + i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.forcemerge.numberOfSegmentsRequiredError', + { defaultMessage: 'A value for number of segments is required.' } + ) + ), + }, + { + validator: ifExistsNumberGreaterThanZero, + }, + ], + serializer: serializers.stringToNumber, + }, + }, + set_priority: { + priority: { + defaultValue: defaultPhaseIndexPriority as any, + label: i18nTexts.editPolicy.setPriorityFieldLabel, validations: [{ validator: ifExistsNumberGreaterThanZero }], - serializer: (v: string): any => (v ? parseInt(v, 10) : undefined), + serializer: serializers.stringToNumber, }, }, }, diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/form_validations.ts b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/form_validations.ts index b937ea2043138c..37ca4e9def340f 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/form_validations.ts +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/form_validations.ts @@ -4,27 +4,19 @@ * you may not use this file except in compliance with the Elastic License. */ -import { i18n } from '@kbn/i18n'; import { fieldValidators, ValidationFunc } from '../../../shared_imports'; -import { i18nTexts } from './components/phases/hot_phase/i18n_texts'; - import { ROLLOVER_FORM_PATHS } from './constants'; -const { numberGreaterThanField } = fieldValidators; +import { i18nTexts } from './i18n_texts'; -export const positiveNumberRequiredMessage = i18n.translate( - 'xpack.indexLifecycleMgmt.editPolicy.numberAboveZeroRequiredError', - { - defaultMessage: 'Only numbers above 0 are allowed.', - } -); +const { numberGreaterThanField } = fieldValidators; export const ifExistsNumberGreaterThanZero: ValidationFunc = (arg) => { if (arg.value) { return numberGreaterThanField({ than: 0, - message: positiveNumberRequiredMessage, + message: i18nTexts.editPolicy.errors.numberGreatThan0Required, })({ ...arg, value: parseInt(arg.value, 10), @@ -54,16 +46,22 @@ export const rolloverThresholdsValidator: ValidationFunc = ({ form }) => { ) ) { fields[ROLLOVER_FORM_PATHS.maxAge].setErrors([ - { validationType: ROLLOVER_EMPTY_VALIDATION, message: i18nTexts.maximumAgeRequiredMessage }, + { + validationType: ROLLOVER_EMPTY_VALIDATION, + message: i18nTexts.editPolicy.errors.maximumAgeRequiredMessage, + }, ]); fields[ROLLOVER_FORM_PATHS.maxDocs].setErrors([ { validationType: ROLLOVER_EMPTY_VALIDATION, - message: i18nTexts.maximumDocumentsRequiredMessage, + message: i18nTexts.editPolicy.errors.maximumDocumentsRequiredMessage, }, ]); fields[ROLLOVER_FORM_PATHS.maxSize].setErrors([ - { validationType: ROLLOVER_EMPTY_VALIDATION, message: i18nTexts.maximumSizeRequiredMessage }, + { + validationType: ROLLOVER_EMPTY_VALIDATION, + message: i18nTexts.editPolicy.errors.maximumSizeRequiredMessage, + }, ]); } else { fields[ROLLOVER_FORM_PATHS.maxAge].clearErrors(ROLLOVER_EMPTY_VALIDATION); diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/i18n_texts.ts b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/i18n_texts.ts index 31bb10b402d27c..1fba69b7634aea 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/i18n_texts.ts +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/i18n_texts.ts @@ -11,5 +11,93 @@ export const i18nTexts = { forceMergeEnabledFieldLabel: i18n.translate('xpack.indexLifecycleMgmt.forcemerge.enableLabel', { defaultMessage: 'Force merge data', }), + maxNumSegmentsFieldLabel: i18n.translate( + 'xpack.indexLifecycleMgmt.forceMerge.numberOfSegmentsLabel', + { + defaultMessage: 'Number of segments', + } + ), + setPriorityFieldLabel: i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.indexPriorityLabel', + { + defaultMessage: 'Index priority (optional)', + } + ), + bestCompressionFieldLabel: i18n.translate( + 'xpack.indexLifecycleMgmt.forcemerge.bestCompressionLabel', + { + defaultMessage: 'Compress stored fields', + } + ), + bestCompressionFieldHelpText: i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.forceMerge.bestCompressionText', + { + defaultMessage: + 'Use higher compression for stored fields at the cost of slower performance.', + } + ), + allocationTypeOptionsFieldLabel: i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.dataTierAllocation.allocationFieldLabel', + { defaultMessage: 'Data tier options' } + ), + allocationNodeAttributeFieldLabel: i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.dataTierAllocation.nodeAllocationFieldLabel', + { + defaultMessage: 'Select a node attribute', + } + ), + errors: { + numberRequired: i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.errors.numberRequiredErrorMessage', + { + defaultMessage: 'A number is required.', + } + ), + numberGreatThan0Required: i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.errors.numberAboveZeroRequiredError', + { + defaultMessage: 'Only numbers above 0 are allowed.', + } + ), + maximumAgeRequiredMessage: i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.errors.maximumAgeMissingError', + { + defaultMessage: 'A maximum age is required.', + } + ), + maximumSizeRequiredMessage: i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.errors.maximumIndexSizeMissingError', + { + defaultMessage: 'A maximum index size is required.', + } + ), + maximumDocumentsRequiredMessage: i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.errors.maximumDocumentsMissingError', + { + defaultMessage: 'Maximum documents is required.', + } + ), + rollOverConfigurationCallout: { + title: i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.errors.rolloverConfigurationError.title', + { + defaultMessage: 'Invalid rollover configuration', + } + ), + body: i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.errors.rolloverConfigurationError.body', + { + defaultMessage: + 'A value for one of maximum size, maximum documents, or maximum age is required.', + } + ), + }, + nonNegativeNumberRequired: i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.errors.nonNegativeNumberRequiredError', + { + defaultMessage: 'Only non-negative numbers are allowed.', + } + ), + }, }, }; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/serializer.ts b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/serializer.ts index e0e1ad44f15571..90e81528f5afe6 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/serializer.ts +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/serializer.ts @@ -4,40 +4,130 @@ * you may not use this file except in compliance with the Elastic License. */ -import { SerializedPolicy } from '../../../../common/types'; +import { isEmpty } from 'lodash'; -import { FormInternal } from './types'; +import { SerializedPolicy, SerializedActionWithAllocation } from '../../../../common/types'; + +import { FormInternal, DataAllocationMetaFields } from './types'; +import { isNumber } from '../../services/policies/policy_serialization'; + +const serializeAllocateAction = ( + { dataTierAllocationType, allocationNodeAttribute }: DataAllocationMetaFields, + newActions: SerializedActionWithAllocation = {}, + originalActions: SerializedActionWithAllocation = {} +): SerializedActionWithAllocation => { + const { allocate, migrate, ...rest } = newActions; + // First copy over all non-allocate and migrate actions. + const actions: SerializedActionWithAllocation = { allocate, migrate, ...rest }; + + switch (dataTierAllocationType) { + case 'node_attrs': + if (allocationNodeAttribute) { + const [name, value] = allocationNodeAttribute.split(':'); + actions.allocate = { + // copy over any other allocate details like "number_of_replicas" + ...actions.allocate, + require: { + [name]: value, + }, + }; + } + + // copy over the original include and exclude values until we can set them in the form. + if (!isEmpty(originalActions?.allocate?.include)) { + actions.allocate = { + ...actions.allocate, + include: { ...originalActions?.allocate?.include }, + }; + } + + if (!isEmpty(originalActions?.allocate?.exclude)) { + actions.allocate = { + ...actions.allocate, + exclude: { ...originalActions?.allocate?.exclude }, + }; + } + break; + case 'none': + actions.migrate = { enabled: false }; + break; + default: + } + return actions; +}; export const createSerializer = (originalPolicy?: SerializedPolicy) => ( data: FormInternal ): SerializedPolicy => { - const { _meta, ...rest } = data; + const { _meta, ...policy } = data; - if (!rest.phases || !rest.phases.hot) { - rest.phases = { hot: { actions: {} } }; + if (!policy.phases || !policy.phases.hot) { + policy.phases = { hot: { actions: {} } }; } - if (rest.phases.hot) { - rest.phases.hot.min_age = originalPolicy?.phases.hot?.min_age ?? '0ms'; + /** + * HOT PHASE SERIALIZATION + */ + if (policy.phases.hot) { + policy.phases.hot.min_age = originalPolicy?.phases.hot?.min_age ?? '0ms'; } - if (rest.phases.hot?.actions) { - if (rest.phases.hot.actions?.rollover && _meta.hot.useRollover) { - if (rest.phases.hot.actions.rollover.max_age) { - rest.phases.hot.actions.rollover.max_age = `${rest.phases.hot.actions.rollover.max_age}${_meta.hot.maxAgeUnit}`; + if (policy.phases.hot?.actions) { + if (policy.phases.hot.actions?.rollover && _meta.hot.useRollover) { + if (policy.phases.hot.actions.rollover.max_age) { + policy.phases.hot.actions.rollover.max_age = `${policy.phases.hot.actions.rollover.max_age}${_meta.hot.maxAgeUnit}`; } - if (rest.phases.hot.actions.rollover.max_size) { - rest.phases.hot.actions.rollover.max_size = `${rest.phases.hot.actions.rollover.max_size}${_meta.hot.maxStorageSizeUnit}`; + if (policy.phases.hot.actions.rollover.max_size) { + policy.phases.hot.actions.rollover.max_size = `${policy.phases.hot.actions.rollover.max_size}${_meta.hot.maxStorageSizeUnit}`; } - if (_meta.hot.bestCompression && rest.phases.hot.actions?.forcemerge) { - rest.phases.hot.actions.forcemerge.index_codec = 'best_compression'; + if (_meta.hot.bestCompression && policy.phases.hot.actions?.forcemerge) { + policy.phases.hot.actions.forcemerge.index_codec = 'best_compression'; } } else { - delete rest.phases.hot.actions?.rollover; + delete policy.phases.hot.actions?.rollover; + } + } + + /** + * WARM PHASE SERIALIZATION + */ + if (policy.phases.warm) { + // If warm phase on rollover is enabled, delete min age field + // An index lifecycle switches to warm phase when rollover occurs, so you cannot specify a warm phase time + // They are mutually exclusive + if (_meta.hot.useRollover && _meta.warm.warmPhaseOnRollover) { + delete policy.phases.warm.min_age; + } else if ( + (!_meta.hot.useRollover || !_meta.warm.warmPhaseOnRollover) && + policy.phases.warm.min_age + ) { + policy.phases.warm.min_age = `${policy.phases.warm.min_age}${_meta.warm.minAgeUnit}`; + } + + policy.phases.warm.actions = serializeAllocateAction( + _meta.warm, + policy.phases.warm.actions, + originalPolicy?.phases.warm?.actions + ); + + if ( + policy.phases.warm.actions.allocate && + !policy.phases.warm.actions.allocate.require && + !isNumber(policy.phases.warm.actions.allocate.number_of_replicas) && + isEmpty(policy.phases.warm.actions.allocate.include) && + isEmpty(policy.phases.warm.actions.allocate.exclude) + ) { + // remove allocate action if it does not define require or number of nodes + // and both include and exclude are empty objects (ES will fail to parse if we don't) + delete policy.phases.warm.actions.allocate; + } + + if (_meta.warm.bestCompression && policy.phases.warm.actions?.forcemerge) { + policy.phases.warm.actions.forcemerge.index_codec = 'best_compression'; } } - return rest; + return policy; }; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/types.ts b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/types.ts index dba56eb8ecbf3e..6fcfbd050c69d0 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/types.ts +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/types.ts @@ -6,6 +6,29 @@ import { SerializedPolicy } from '../../../../common/types'; +export type DataTierAllocationType = 'node_roles' | 'node_attrs' | 'none'; + +export interface DataAllocationMetaFields { + dataTierAllocationType: DataTierAllocationType; + allocationNodeAttribute?: string; +} + +interface HotPhaseMetaFields { + useRollover: boolean; + forceMergeEnabled: boolean; + bestCompression: boolean; + maxStorageSizeUnit?: string; + maxAgeUnit?: string; +} + +interface WarmPhaseMetaFields extends DataAllocationMetaFields { + enabled: boolean; + forceMergeEnabled: boolean; + bestCompression: boolean; + warmPhaseOnRollover: boolean; + minAgeUnit?: string; +} + /** * Describes the shape of data after deserialization. */ @@ -15,12 +38,7 @@ export interface FormInternal extends SerializedPolicy { * certain form fields which affects what is ultimately serialized. */ _meta: { - hot: { - useRollover: boolean; - forceMergeEnabled: boolean; - bestCompression: boolean; - maxStorageSizeUnit?: string; - maxAgeUnit?: string; - }; + hot: HotPhaseMetaFields; + warm: WarmPhaseMetaFields; }; } diff --git a/x-pack/plugins/index_lifecycle_management/public/application/services/policies/cold_phase.ts b/x-pack/plugins/index_lifecycle_management/public/application/services/policies/cold_phase.ts index 9f5f603fbc5640..faf3954f93fd8c 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/services/policies/cold_phase.ts +++ b/x-pack/plugins/index_lifecycle_management/public/application/services/policies/cold_phase.ts @@ -13,7 +13,7 @@ import { PhaseValidationErrors, positiveNumberRequiredMessage, } from './policy_validation'; -import { determineDataTierAllocationType } from '../../lib'; +import { determineDataTierAllocationTypeLegacy } from '../../lib'; import { serializePhaseWithAllocation } from './shared'; export const coldPhaseInitialization: ColdPhase = { @@ -35,10 +35,8 @@ export const coldPhaseFromES = (phaseSerialized?: SerializedColdPhase): ColdPhas phase.phaseEnabled = true; - if (phaseSerialized.actions.allocate) { - phase.dataTierAllocationType = determineDataTierAllocationType( - phaseSerialized.actions.allocate - ); + if (phaseSerialized.actions) { + phase.dataTierAllocationType = determineDataTierAllocationTypeLegacy(phaseSerialized.actions); } if (phaseSerialized.min_age) { diff --git a/x-pack/plugins/index_lifecycle_management/public/application/services/policies/policy_serialization.test.ts b/x-pack/plugins/index_lifecycle_management/public/application/services/policies/policy_serialization.test.ts index 5c7f04986827b4..0be6ab35217360 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/services/policies/policy_serialization.test.ts +++ b/x-pack/plugins/index_lifecycle_management/public/application/services/policies/policy_serialization.test.ts @@ -7,7 +7,7 @@ // eslint-disable-next-line no-restricted-imports import cloneDeep from 'lodash/cloneDeep'; import { deserializePolicy, legacySerializePolicy } from './policy_serialization'; -import { defaultNewColdPhase, defaultNewDeletePhase, defaultNewWarmPhase } from '../../constants'; +import { defaultNewColdPhase, defaultNewDeletePhase } from '../../constants'; import { DataTierAllocationType } from '../../../../common/types'; import { coldPhaseInitialization } from './cold_phase'; @@ -18,13 +18,6 @@ describe('Policy serialization', () => { { name: 'test', phases: { - warm: { - ...defaultNewWarmPhase, - dataTierAllocationType: 'default', - // These selected attrs should be ignored - selectedNodeAttrs: 'another:thing', - phaseEnabled: true, - }, cold: { ...defaultNewColdPhase, dataTierAllocationType: 'default', @@ -38,9 +31,6 @@ describe('Policy serialization', () => { name: 'test', phases: { hot: { actions: {} }, - warm: { - actions: { allocate: { include: {}, exclude: {}, require: { something: 'here' } } }, - }, cold: { actions: { allocate: { include: {}, exclude: {}, require: { something: 'here' } } }, }, @@ -50,13 +40,6 @@ describe('Policy serialization', () => { ).toEqual({ name: 'test', phases: { - warm: { - actions: { - set_priority: { - priority: 50, - }, - }, - }, cold: { actions: { set_priority: { @@ -75,12 +58,6 @@ describe('Policy serialization', () => { { name: 'test', phases: { - warm: { - ...defaultNewWarmPhase, - dataTierAllocationType: 'custom', - selectedNodeAttrs: 'another:thing', - phaseEnabled: true, - }, cold: { ...defaultNewColdPhase, dataTierAllocationType: 'custom', @@ -94,15 +71,6 @@ describe('Policy serialization', () => { name: 'test', phases: { hot: { actions: {} }, - warm: { - actions: { - allocate: { - include: { keep: 'this' }, - exclude: { keep: 'this' }, - require: { something: 'here' }, - }, - }, - }, cold: { actions: { allocate: { @@ -118,20 +86,6 @@ describe('Policy serialization', () => { ).toEqual({ name: 'test', phases: { - warm: { - actions: { - allocate: { - include: { keep: 'this' }, - exclude: { keep: 'this' }, - require: { - another: 'thing', - }, - }, - set_priority: { - priority: 50, - }, - }, - }, cold: { actions: { allocate: { @@ -157,12 +111,6 @@ describe('Policy serialization', () => { { name: 'test', phases: { - warm: { - ...defaultNewWarmPhase, - dataTierAllocationType: 'custom', - selectedNodeAttrs: '', - phaseEnabled: true, - }, cold: { ...defaultNewColdPhase, dataTierAllocationType: 'custom', @@ -176,9 +124,6 @@ describe('Policy serialization', () => { name: 'test', phases: { hot: { actions: {} }, - warm: { - actions: { allocate: { include: {}, exclude: {}, require: { something: 'here' } } }, - }, cold: { actions: { allocate: { include: {}, exclude: {}, require: { something: 'here' } } }, }, @@ -189,14 +134,6 @@ describe('Policy serialization', () => { // There should be no allocation action in any phases... name: 'test', phases: { - warm: { - actions: { - allocate: { include: {}, exclude: {}, require: { something: 'here' } }, - set_priority: { - priority: 50, - }, - }, - }, cold: { actions: { allocate: { include: {}, exclude: {}, require: { something: 'here' } }, @@ -216,12 +153,6 @@ describe('Policy serialization', () => { { name: 'test', phases: { - warm: { - ...defaultNewWarmPhase, - dataTierAllocationType: 'none', - selectedNodeAttrs: 'ignore:this', - phaseEnabled: true, - }, cold: { ...defaultNewColdPhase, dataTierAllocationType: 'none', @@ -235,9 +166,6 @@ describe('Policy serialization', () => { name: 'test', phases: { hot: { actions: {} }, - warm: { - actions: { allocate: { include: {}, exclude: {}, require: { something: 'here' } } }, - }, cold: { actions: { allocate: { include: {}, exclude: {}, require: { something: 'here' } } }, }, @@ -248,16 +176,6 @@ describe('Policy serialization', () => { // There should be no allocation action in any phases... name: 'test', phases: { - warm: { - actions: { - migrate: { - enabled: false, - }, - set_priority: { - priority: 50, - }, - }, - }, cold: { actions: { migrate: { @@ -277,9 +195,6 @@ describe('Policy serialization', () => { const originalPolicy = { name: 'test', phases: { - warm: { - actions: { allocate: { include: {}, exclude: {}, require: { something: 'here' } } }, - }, cold: { actions: { allocate: { include: {}, exclude: {}, require: { something: 'here' } } }, }, @@ -291,12 +206,6 @@ describe('Policy serialization', () => { const deserializedPolicy = { name: 'test', phases: { - warm: { - ...defaultNewWarmPhase, - dataTierAllocationType: 'none' as DataTierAllocationType, - selectedNodeAttrs: 'ignore:this', - phaseEnabled: true, - }, cold: { ...defaultNewColdPhase, dataTierAllocationType: 'none' as DataTierAllocationType, @@ -308,10 +217,6 @@ describe('Policy serialization', () => { }, }; - legacySerializePolicy(deserializedPolicy, originalPolicy); - deserializedPolicy.phases.warm.dataTierAllocationType = 'custom'; - legacySerializePolicy(deserializedPolicy, originalPolicy); - deserializedPolicy.phases.warm.dataTierAllocationType = 'default'; legacySerializePolicy(deserializedPolicy, originalPolicy); expect(originalPolicy).toEqual(originalClone); }); @@ -322,13 +227,6 @@ describe('Policy serialization', () => { { name: 'test', phases: { - warm: { - ...defaultNewWarmPhase, - phaseEnabled: true, - forceMergeEnabled: true, - selectedForceMergeSegments: '1', - bestCompressionEnabled: true, - }, cold: { ...defaultNewColdPhase, }, @@ -344,19 +242,7 @@ describe('Policy serialization', () => { ) ).toEqual({ name: 'test', - phases: { - warm: { - actions: { - forcemerge: { - max_num_segments: 1, - index_codec: 'best_compression', - }, - set_priority: { - priority: 50, - }, - }, - }, - }, + phases: {}, }); }); @@ -384,31 +270,12 @@ describe('Policy serialization', () => { }, }, }, - warm: { - actions: { - forcemerge: { - max_num_segments: 1, - index_codec: 'best_compression', - }, - set_priority: { - priority: 50, - }, - }, - }, }, }, }) ).toEqual({ name: 'test', phases: { - warm: { - ...defaultNewWarmPhase, - warmPhaseOnRollover: false, - phaseEnabled: true, - forceMergeEnabled: true, - selectedForceMergeSegments: '1', - bestCompressionEnabled: true, - }, cold: { ...coldPhaseInitialization, }, @@ -423,13 +290,6 @@ describe('Policy serialization', () => { { name: 'test', phases: { - warm: { - ...defaultNewWarmPhase, - phaseEnabled: true, - forceMergeEnabled: true, - selectedForceMergeSegments: '1', - bestCompressionEnabled: false, - }, cold: { ...defaultNewColdPhase, }, @@ -438,32 +298,12 @@ describe('Policy serialization', () => { }, { name: 'test', - phases: { - warm: { - actions: { - forcemerge: { - max_num_segments: 1, - index_codec: 'best_compression', - }, - }, - }, - }, + phases: {}, } ) ).toEqual({ name: 'test', - phases: { - warm: { - actions: { - forcemerge: { - max_num_segments: 1, - }, - set_priority: { - priority: 50, - }, - }, - }, - }, + phases: {}, }); }); }); diff --git a/x-pack/plugins/index_lifecycle_management/public/application/services/policies/policy_serialization.ts b/x-pack/plugins/index_lifecycle_management/public/application/services/policies/policy_serialization.ts index 0dce7efce46236..32c7e698b09203 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/services/policies/policy_serialization.ts +++ b/x-pack/plugins/index_lifecycle_management/public/application/services/policies/policy_serialization.ts @@ -9,11 +9,9 @@ import { LegacyPolicy, PolicyFromES, SerializedPolicy } from '../../../../common import { defaultNewColdPhase, defaultNewDeletePhase, - defaultNewWarmPhase, serializedPhaseInitialization, } from '../../constants'; -import { warmPhaseFromES, warmPhaseToES } from './warm_phase'; import { coldPhaseFromES, coldPhaseToES } from './cold_phase'; import { deletePhaseFromES, deletePhaseToES } from './delete_phase'; @@ -48,7 +46,6 @@ export const initializeNewPolicy = (newPolicyName: string = ''): LegacyPolicy => return { name: newPolicyName, phases: { - warm: { ...defaultNewWarmPhase }, cold: { ...defaultNewColdPhase }, delete: { ...defaultNewDeletePhase }, }, @@ -64,7 +61,6 @@ export const deserializePolicy = (policy: PolicyFromES): LegacyPolicy => { return { name, phases: { - warm: warmPhaseFromES(phases.warm), cold: coldPhaseFromES(phases.cold), delete: deletePhaseFromES(phases.delete), }, @@ -82,9 +78,6 @@ export const legacySerializePolicy = ( name: policy.name, phases: {}, } as SerializedPolicy; - if (policy.phases.warm.phaseEnabled) { - serializedPolicy.phases.warm = warmPhaseToES(policy.phases.warm, originalEsPolicy.phases.warm); - } if (policy.phases.cold.phaseEnabled) { serializedPolicy.phases.cold = coldPhaseToES(policy.phases.cold, originalEsPolicy.phases.cold); diff --git a/x-pack/plugins/index_lifecycle_management/public/application/services/policies/policy_validation.ts b/x-pack/plugins/index_lifecycle_management/public/application/services/policies/policy_validation.ts index eeceb97c409f55..a113cb68a23496 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/services/policies/policy_validation.ts +++ b/x-pack/plugins/index_lifecycle_management/public/application/services/policies/policy_validation.ts @@ -5,14 +5,7 @@ */ import { i18n } from '@kbn/i18n'; -import { - ColdPhase, - DeletePhase, - LegacyPolicy, - PolicyFromES, - WarmPhase, -} from '../../../../common/types'; -import { validateWarmPhase } from './warm_phase'; +import { ColdPhase, DeletePhase, LegacyPolicy, PolicyFromES } from '../../../../common/types'; import { validateColdPhase } from './cold_phase'; import { validateDeletePhase } from './delete_phase'; @@ -89,7 +82,6 @@ export type PhaseValidationErrors = { }; export interface ValidationErrors { - warm: PhaseValidationErrors; cold: PhaseValidationErrors; delete: PhaseValidationErrors; policyName: string[]; @@ -128,19 +120,16 @@ export const validatePolicy = ( } } - const warmPhaseErrors = validateWarmPhase(policy.phases.warm); const coldPhaseErrors = validateColdPhase(policy.phases.cold); const deletePhaseErrors = validateDeletePhase(policy.phases.delete); const isValid = policyNameErrors.length === 0 && - Object.keys(warmPhaseErrors).length === 0 && Object.keys(coldPhaseErrors).length === 0 && Object.keys(deletePhaseErrors).length === 0; return [ isValid, { policyName: [...policyNameErrors], - warm: warmPhaseErrors, cold: coldPhaseErrors, delete: deletePhaseErrors, }, @@ -156,9 +145,6 @@ export const findFirstError = (errors?: ValidationErrors): string | undefined => return propertyof('policyName'); } - if (Object.keys(errors.warm).length > 0) { - return `${propertyof('warm')}.${Object.keys(errors.warm)[0]}`; - } if (Object.keys(errors.cold).length > 0) { return `${propertyof('cold')}.${Object.keys(errors.cold)[0]}`; } diff --git a/x-pack/plugins/index_lifecycle_management/public/application/services/policies/warm_phase.ts b/x-pack/plugins/index_lifecycle_management/public/application/services/policies/warm_phase.ts deleted file mode 100644 index 436e5a222f86d4..00000000000000 --- a/x-pack/plugins/index_lifecycle_management/public/application/services/policies/warm_phase.ts +++ /dev/null @@ -1,230 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { isEmpty } from 'lodash'; -import { AllocateAction, WarmPhase, SerializedWarmPhase } from '../../../../common/types'; -import { serializedPhaseInitialization } from '../../constants'; -import { isNumber, splitSizeAndUnits } from './policy_serialization'; - -import { - numberRequiredMessage, - PhaseValidationErrors, - positiveNumberRequiredMessage, - positiveNumbersAboveZeroErrorMessage, -} from './policy_validation'; - -import { determineDataTierAllocationType } from '../../lib'; -import { serializePhaseWithAllocation } from './shared'; - -const warmPhaseInitialization: WarmPhase = { - phaseEnabled: false, - warmPhaseOnRollover: false, - selectedMinimumAge: '0', - selectedMinimumAgeUnits: 'd', - selectedNodeAttrs: '', - selectedReplicaCount: '', - shrinkEnabled: false, - selectedPrimaryShardCount: '', - forceMergeEnabled: false, - selectedForceMergeSegments: '', - bestCompressionEnabled: false, - phaseIndexPriority: '', - dataTierAllocationType: 'default', -}; - -export const warmPhaseFromES = (phaseSerialized?: SerializedWarmPhase): WarmPhase => { - const phase: WarmPhase = { ...warmPhaseInitialization }; - - if (phaseSerialized === undefined || phaseSerialized === null) { - return phase; - } - - phase.phaseEnabled = true; - - if (phaseSerialized.actions.allocate) { - phase.dataTierAllocationType = determineDataTierAllocationType( - phaseSerialized.actions.allocate - ); - } - - if (phaseSerialized.min_age) { - if (phaseSerialized.min_age === '0ms') { - phase.warmPhaseOnRollover = true; - } else { - const { size: minAge, units: minAgeUnits } = splitSizeAndUnits(phaseSerialized.min_age); - phase.selectedMinimumAge = minAge; - phase.selectedMinimumAgeUnits = minAgeUnits; - } - } - if (phaseSerialized.actions) { - const actions = phaseSerialized.actions; - if (actions.allocate) { - const allocate = actions.allocate; - if (allocate.require) { - Object.entries(allocate.require).forEach((entry) => { - phase.selectedNodeAttrs = entry.join(':'); - }); - if (allocate.number_of_replicas) { - phase.selectedReplicaCount = allocate.number_of_replicas.toString(); - } - } - } - - if (actions.forcemerge) { - const forcemerge = actions.forcemerge; - phase.forceMergeEnabled = true; - phase.selectedForceMergeSegments = forcemerge.max_num_segments.toString(); - // only accepted value for index_codec - phase.bestCompressionEnabled = forcemerge.index_codec === 'best_compression'; - } - - if (actions.shrink) { - phase.shrinkEnabled = true; - phase.selectedPrimaryShardCount = actions.shrink.number_of_shards - ? actions.shrink.number_of_shards.toString() - : ''; - } - - if (actions.set_priority) { - phase.phaseIndexPriority = actions.set_priority.priority - ? actions.set_priority.priority.toString() - : ''; - } - } - return phase; -}; - -export const warmPhaseToES = ( - phase: WarmPhase, - originalEsPhase?: SerializedWarmPhase -): SerializedWarmPhase => { - if (!originalEsPhase) { - originalEsPhase = { ...serializedPhaseInitialization }; - } - - const esPhase = { ...originalEsPhase }; - - if (isNumber(phase.selectedMinimumAge)) { - esPhase.min_age = `${phase.selectedMinimumAge}${phase.selectedMinimumAgeUnits}`; - } - - // If warm phase on rollover is enabled, delete min age field - // An index lifecycle switches to warm phase when rollover occurs, so you cannot specify a warm phase time - // They are mutually exclusive - if (phase.warmPhaseOnRollover) { - delete esPhase.min_age; - } - - esPhase.actions = serializePhaseWithAllocation(phase, esPhase.actions); - - if (isNumber(phase.selectedReplicaCount)) { - esPhase.actions.allocate = esPhase.actions.allocate || ({} as AllocateAction); - esPhase.actions.allocate.number_of_replicas = parseInt(phase.selectedReplicaCount, 10); - } else { - if (esPhase.actions.allocate) { - delete esPhase.actions.allocate.number_of_replicas; - } - } - - if ( - esPhase.actions.allocate && - !esPhase.actions.allocate.require && - !isNumber(esPhase.actions.allocate.number_of_replicas) && - isEmpty(esPhase.actions.allocate.include) && - isEmpty(esPhase.actions.allocate.exclude) - ) { - // remove allocate action if it does not define require or number of nodes - // and both include and exclude are empty objects (ES will fail to parse if we don't) - delete esPhase.actions.allocate; - } - - if (phase.forceMergeEnabled) { - esPhase.actions.forcemerge = { - max_num_segments: parseInt(phase.selectedForceMergeSegments, 10), - }; - if (phase.bestCompressionEnabled) { - // only accepted value for index_codec - esPhase.actions.forcemerge.index_codec = 'best_compression'; - } - } else { - delete esPhase.actions.forcemerge; - } - - if (phase.shrinkEnabled && isNumber(phase.selectedPrimaryShardCount)) { - esPhase.actions.shrink = { - number_of_shards: parseInt(phase.selectedPrimaryShardCount, 10), - }; - } else { - delete esPhase.actions.shrink; - } - - if (isNumber(phase.phaseIndexPriority)) { - esPhase.actions.set_priority = { - priority: parseInt(phase.phaseIndexPriority, 10), - }; - } else { - delete esPhase.actions.set_priority; - } - - return esPhase; -}; - -export const validateWarmPhase = (phase: WarmPhase): PhaseValidationErrors => { - if (!phase.phaseEnabled) { - return {}; - } - - const phaseErrors = {} as PhaseValidationErrors; - - // index priority is optional, but if it's set, it needs to be a positive number - if (phase.phaseIndexPriority) { - if (!isNumber(phase.phaseIndexPriority)) { - phaseErrors.phaseIndexPriority = [numberRequiredMessage]; - } else if (parseInt(phase.phaseIndexPriority, 10) < 0) { - phaseErrors.phaseIndexPriority = [positiveNumberRequiredMessage]; - } - } - - // if warm phase on rollover is disabled, min age needs to be a positive number - if (!phase.warmPhaseOnRollover) { - if (!isNumber(phase.selectedMinimumAge)) { - phaseErrors.selectedMinimumAge = [numberRequiredMessage]; - } else if (parseInt(phase.selectedMinimumAge, 10) < 0) { - phaseErrors.selectedMinimumAge = [positiveNumberRequiredMessage]; - } - } - - // if forcemerge is enabled, force merge segments needs to be a number above zero - if (phase.forceMergeEnabled) { - if (!isNumber(phase.selectedForceMergeSegments)) { - phaseErrors.selectedForceMergeSegments = [numberRequiredMessage]; - } else if (parseInt(phase.selectedForceMergeSegments, 10) < 1) { - phaseErrors.selectedForceMergeSegments = [positiveNumbersAboveZeroErrorMessage]; - } - } - - // if shrink is enabled, primary shard count needs to be a number above zero - if (phase.shrinkEnabled) { - if (!isNumber(phase.selectedPrimaryShardCount)) { - phaseErrors.selectedPrimaryShardCount = [numberRequiredMessage]; - } else if (parseInt(phase.selectedPrimaryShardCount, 10) < 1) { - phaseErrors.selectedPrimaryShardCount = [positiveNumbersAboveZeroErrorMessage]; - } - } - - // replica count is optional, but if it's set, it needs to be a positive number - if (phase.selectedReplicaCount) { - if (!isNumber(phase.selectedReplicaCount)) { - phaseErrors.selectedReplicaCount = [numberRequiredMessage]; - } else if (parseInt(phase.selectedReplicaCount, 10) < 0) { - phaseErrors.selectedReplicaCount = [numberRequiredMessage]; - } - } - - return { - ...phaseErrors, - }; -}; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/services/ui_metric.test.ts b/x-pack/plugins/index_lifecycle_management/public/application/services/ui_metric.test.ts index 81eb1c8cad1358..c77e3d22f0e37c 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/services/ui_metric.test.ts +++ b/x-pack/plugins/index_lifecycle_management/public/application/services/ui_metric.test.ts @@ -9,8 +9,8 @@ import { UIM_CONFIG_WARM_PHASE, UIM_CONFIG_SET_PRIORITY, UIM_CONFIG_FREEZE_INDEX, - defaultNewWarmPhase, defaultNewColdPhase, + defaultPhaseIndexPriority, } from '../constants/'; import { getUiMetricsForPhases } from './ui_metric'; @@ -38,7 +38,7 @@ describe('getUiMetricsForPhases', () => { min_age: '0ms', actions: { set_priority: { - priority: parseInt(defaultNewWarmPhase.phaseIndexPriority, 10), + priority: parseInt(defaultPhaseIndexPriority, 10), }, }, }, @@ -53,7 +53,7 @@ describe('getUiMetricsForPhases', () => { min_age: '0ms', actions: { set_priority: { - priority: parseInt(defaultNewWarmPhase.phaseIndexPriority, 10) + 1, + priority: parseInt(defaultPhaseIndexPriority, 10) + 1, }, }, }, diff --git a/x-pack/plugins/index_lifecycle_management/public/application/services/ui_metric.ts b/x-pack/plugins/index_lifecycle_management/public/application/services/ui_metric.ts index ea5c5619da5899..305b35b23e4d8b 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/services/ui_metric.ts +++ b/x-pack/plugins/index_lifecycle_management/public/application/services/ui_metric.ts @@ -14,8 +14,8 @@ import { UIM_CONFIG_SET_PRIORITY, UIM_CONFIG_WARM_PHASE, defaultNewColdPhase, - defaultNewWarmPhase, defaultSetPriority, + defaultPhaseIndexPriority, } from '../constants'; import { Phases } from '../../../common/types'; @@ -50,8 +50,7 @@ export function getUiMetricsForPhases(phases: Phases): string[] { const isWarmPhasePriorityChanged = phases.warm && phases.warm.actions.set_priority && - phases.warm.actions.set_priority.priority !== - parseInt(defaultNewWarmPhase.phaseIndexPriority, 10); + phases.warm.actions.set_priority.priority !== parseInt(defaultPhaseIndexPriority, 10); const isColdPhasePriorityChanged = phases.cold && diff --git a/x-pack/plugins/index_lifecycle_management/public/shared_imports.ts b/x-pack/plugins/index_lifecycle_management/public/shared_imports.ts index dc3e1b1d1b62da..023aeba57aa7a6 100644 --- a/x-pack/plugins/index_lifecycle_management/public/shared_imports.ts +++ b/x-pack/plugins/index_lifecycle_management/public/shared_imports.ts @@ -26,6 +26,7 @@ export { ToggleField, NumericField, SelectField, + SuperSelectField, } from '../../../../src/plugins/es_ui_shared/static/forms/components'; export { KibanaContextProvider } from '../../../../src/plugins/kibana_react/public'; diff --git a/x-pack/plugins/index_lifecycle_management/server/routes/api/nodes/register_list_route.ts b/x-pack/plugins/index_lifecycle_management/server/routes/api/nodes/register_list_route.ts index 53955d93c1e09a..0603ebf6eebe01 100644 --- a/x-pack/plugins/index_lifecycle_management/server/routes/api/nodes/register_list_route.ts +++ b/x-pack/plugins/index_lifecycle_management/server/routes/api/nodes/register_list_route.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { ListNodesRouteResponse, NodeDataRole } from '../../../../common/types'; +import { ListNodesRouteResponse, DataTierRole } from '../../../../common/types'; import { RouteDependencies } from '../../../types'; import { addBasePath } from '../../../services'; @@ -39,10 +39,10 @@ export function convertSettingsIntoLists( } } - const dataRoles = nodeSettings.roles.filter((r) => r.startsWith('data')) as NodeDataRole[]; + const dataRoles = nodeSettings.roles.filter((r) => r.startsWith('data')) as DataTierRole[]; for (const role of dataRoles) { - accum.nodesByRoles[role as NodeDataRole] = accum.nodesByRoles[role] ?? []; - accum.nodesByRoles[role as NodeDataRole]!.push(nodeId); + accum.nodesByRoles[role as DataTierRole] = accum.nodesByRoles[role] ?? []; + accum.nodesByRoles[role as DataTierRole]!.push(nodeId); } // If we detect a single node using legacy "data:true" setting we know we are not using data roles for diff --git a/x-pack/plugins/infra/server/lib/adapters/framework/kibana_framework_adapter.ts b/x-pack/plugins/infra/server/lib/adapters/framework/kibana_framework_adapter.ts index 2dcab5b49dcdb6..2d84e36f3a3ac9 100644 --- a/x-pack/plugins/infra/server/lib/adapters/framework/kibana_framework_adapter.ts +++ b/x-pack/plugins/infra/server/lib/adapters/framework/kibana_framework_adapter.ts @@ -26,7 +26,6 @@ import { RequestHandlerContext, KibanaResponseFactory, RouteMethod, - LegacyAPICaller, } from '../../../../../../../src/core/server'; import { RequestHandler } from '../../../../../../../src/core/server'; import { InfraConfig } from '../../../plugin'; @@ -218,11 +217,7 @@ export class KibanaFramework { } public getIndexPatternsService(requestContext: RequestHandlerContext): IndexPatternsFetcher { - return new IndexPatternsFetcher((...rest: Parameters) => { - rest[1] = rest[1] || {}; - rest[1].allowNoIndices = true; - return requestContext.core.elasticsearch.legacy.client.callAsCurrentUser(...rest); - }); + return new IndexPatternsFetcher(requestContext.core.elasticsearch.client.asCurrentUser, true); } public getSpaceId(request: KibanaRequest): string { diff --git a/x-pack/plugins/infra/server/routes/metrics_explorer/lib/convert_metric_to_metrics_api_metric.ts b/x-pack/plugins/infra/server/routes/metrics_explorer/lib/convert_metric_to_metrics_api_metric.ts index 93948a8b8797e9..c4f2c5f7981d40 100644 --- a/x-pack/plugins/infra/server/routes/metrics_explorer/lib/convert_metric_to_metrics_api_metric.ts +++ b/x-pack/plugins/infra/server/routes/metrics_explorer/lib/convert_metric_to_metrics_api_metric.ts @@ -9,7 +9,7 @@ import { MetricsAPIMetric, MetricsExplorerMetric } from '../../../../common/http export const convertMetricToMetricsAPIMetric = ( metric: MetricsExplorerMetric, index: number -): MetricsAPIMetric => { +): MetricsAPIMetric | undefined => { const id = `metric_${index}`; if (metric.aggregation === 'rate' && metric.field) { return { @@ -44,19 +44,21 @@ export const convertMetricToMetricsAPIMetric = ( }; } - return { - id, - aggregations: { - [id]: { - bucket_script: { - buckets_path: { count: '_count' }, - script: { - source: 'count * 1', - lang: 'expression', + if (metric.aggregation === 'count') { + return { + id, + aggregations: { + [id]: { + bucket_script: { + buckets_path: { count: '_count' }, + script: { + source: 'count * 1', + lang: 'expression', + }, + gap_policy: 'skip', }, - gap_policy: 'skip', }, }, - }, - }; + }; + } }; diff --git a/x-pack/plugins/infra/server/routes/metrics_explorer/lib/convert_request_to_metrics_api_options.test.ts b/x-pack/plugins/infra/server/routes/metrics_explorer/lib/convert_request_to_metrics_api_options.test.ts index 4c423aee347e9e..887f464b1a5644 100644 --- a/x-pack/plugins/infra/server/routes/metrics_explorer/lib/convert_request_to_metrics_api_options.test.ts +++ b/x-pack/plugins/infra/server/routes/metrics_explorer/lib/convert_request_to_metrics_api_options.test.ts @@ -120,4 +120,16 @@ describe('convertRequestToMetricsAPIOptions', () => { metrics: [], }); }); + + it('should work with empty field', () => { + expect( + convertRequestToMetricsAPIOptions({ + ...BASE_REQUEST, + metrics: [{ aggregation: 'avg' }], + }) + ).toEqual({ + ...BASE_METRICS_UI_OPTIONS, + metrics: [], + }); + }); }); diff --git a/x-pack/plugins/infra/server/routes/metrics_explorer/lib/convert_request_to_metrics_api_options.ts b/x-pack/plugins/infra/server/routes/metrics_explorer/lib/convert_request_to_metrics_api_options.ts index 2dd00c4aed59cc..4bb6d8f55a11a4 100644 --- a/x-pack/plugins/infra/server/routes/metrics_explorer/lib/convert_request_to_metrics_api_options.ts +++ b/x-pack/plugins/infra/server/routes/metrics_explorer/lib/convert_request_to_metrics_api_options.ts @@ -15,7 +15,9 @@ import { convertMetricToMetricsAPIMetric } from './convert_metric_to_metrics_api export const convertRequestToMetricsAPIOptions = ( options: MetricsExplorerRequestBody ): MetricsAPIRequest => { - const metrics = options.metrics.map(convertMetricToMetricsAPIMetric); + const metrics = options.metrics + .map(convertMetricToMetricsAPIMetric) + .filter((m: M): m is NonNullable => !!m); const { limit, timerange, indexPattern } = options; const metricsApiOptions: MetricsAPIRequest = { diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/details_page/components/package_policies/package_policies_table.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/details_page/components/package_policies/package_policies_table.tsx index fa9ce249354297..af4c2f78f14a2f 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/details_page/components/package_policies/package_policies_table.tsx +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/details_page/components/package_policies/package_policies_table.tsx @@ -22,7 +22,6 @@ import { useCapabilities, useLink } from '../../../../../hooks'; import { useAgentPolicyRefresh } from '../../hooks'; interface InMemoryPackagePolicy extends PackagePolicy { - inputTypes: string[]; packageName?: string; packageTitle?: string; packageVersion?: string; @@ -56,11 +55,7 @@ export const PackagePoliciesTable: React.FunctionComponent = ({ // With the package policies provided on input, generate the list of package policies // used in the InMemoryTable (flattens some values for search) as well as // the list of options that will be used in the filters dropdowns - const [packagePolicies, namespaces, inputTypes] = useMemo((): [ - InMemoryPackagePolicy[], - FilterOption[], - FilterOption[] - ] => { + const [packagePolicies, namespaces] = useMemo((): [InMemoryPackagePolicy[], FilterOption[]] => { const namespacesValues: string[] = []; const inputTypesValues: string[] = []; const mappedPackagePolicies = originalPackagePolicies.map( @@ -69,13 +64,8 @@ export const PackagePoliciesTable: React.FunctionComponent = ({ namespacesValues.push(packagePolicy.namespace); } - const dsInputTypes: string[] = []; - - dsInputTypes.sort(stringSortAscending); - return { ...packagePolicy, - inputTypes: dsInputTypes, packageName: packagePolicy.package?.name ?? '', packageTitle: packagePolicy.package?.title ?? '', packageVersion: packagePolicy.package?.version ?? '', @@ -86,11 +76,7 @@ export const PackagePoliciesTable: React.FunctionComponent = ({ namespacesValues.sort(stringSortAscending); inputTypesValues.sort(stringSortAscending); - return [ - mappedPackagePolicies, - namespacesValues.map(toFilterOption), - inputTypesValues.map(toFilterOption), - ]; + return [mappedPackagePolicies, namespacesValues.map(toFilterOption)]; }, [originalPackagePolicies]); const columns = useMemo( @@ -273,13 +259,7 @@ export const PackagePoliciesTable: React.FunctionComponent = ({ name: 'Namespace', options: namespaces, multiSelect: 'or', - }, - { - type: 'field_value_selection', - field: 'inputTypes', - name: 'Input types', - options: inputTypes, - multiSelect: 'or', + operator: 'exact', }, ], }} diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/data_stream/list_page/index.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/data_stream/list_page/index.tsx index bb109d766c50ae..4e32fa0bbc1b95 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/data_stream/list_page/index.tsx +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/data_stream/list_page/index.tsx @@ -182,7 +182,12 @@ export const DataStreamListPage: React.FunctionComponent<{}> = () => { [] ); - const filterOptions: { [key: string]: string[] } = { + const filterOptions: { + [key: string]: Array<{ + value: string; + name: string; + }>; + } = { dataset: [], type: [], namespace: [], @@ -190,21 +195,37 @@ export const DataStreamListPage: React.FunctionComponent<{}> = () => { }; if (dataStreamsData && dataStreamsData.data_streams.length) { + const dataValues: { + [key: string]: string[]; + } = { + dataset: [], + type: [], + namespace: [], + package: [], + }; dataStreamsData.data_streams.forEach((stream) => { const { dataset, type, namespace, package: pkg } = stream; - if (!filterOptions.dataset.includes(dataset)) { - filterOptions.dataset.push(dataset); + if (!dataValues.dataset.includes(dataset)) { + dataValues.dataset.push(dataset); } - if (!filterOptions.type.includes(type)) { - filterOptions.type.push(type); + if (!dataValues.type.includes(type)) { + dataValues.type.push(type); } - if (!filterOptions.namespace.includes(namespace)) { - filterOptions.namespace.push(namespace); + if (!dataValues.namespace.includes(namespace)) { + dataValues.namespace.push(namespace); } - if (!filterOptions.package.includes(pkg)) { - filterOptions.package.push(pkg); + if (!dataValues.package.includes(pkg)) { + dataValues.package.push(pkg); } }); + for (const field in dataValues) { + if (filterOptions[field]) { + filterOptions[field] = dataValues[field].sort().map((option) => ({ + value: option, + name: option, + })); + } + } } return ( @@ -266,10 +287,8 @@ export const DataStreamListPage: React.FunctionComponent<{}> = () => { defaultMessage: 'Dataset', }), multiSelect: 'or', - options: filterOptions.dataset.map((option) => ({ - value: option, - name: option, - })), + operator: 'exact', + options: filterOptions.dataset, }, { type: 'field_value_selection', @@ -278,10 +297,8 @@ export const DataStreamListPage: React.FunctionComponent<{}> = () => { defaultMessage: 'Type', }), multiSelect: 'or', - options: filterOptions.type.map((option) => ({ - value: option, - name: option, - })), + operator: 'exact', + options: filterOptions.type, }, { type: 'field_value_selection', @@ -290,10 +307,8 @@ export const DataStreamListPage: React.FunctionComponent<{}> = () => { defaultMessage: 'Namespace', }), multiSelect: 'or', - options: filterOptions.namespace.map((option) => ({ - value: option, - name: option, - })), + operator: 'exact', + options: filterOptions.namespace, }, { type: 'field_value_selection', @@ -302,10 +317,8 @@ export const DataStreamListPage: React.FunctionComponent<{}> = () => { defaultMessage: 'Integration', }), multiSelect: 'or', - options: filterOptions.package.map((option) => ({ - value: option, - name: option, - })), + operator: 'exact', + options: filterOptions.package, }, ], }} diff --git a/x-pack/plugins/ingest_manager/server/services/agent_policy.ts b/x-pack/plugins/ingest_manager/server/services/agent_policy.ts index 19d69a33788c62..b003d16d379ca9 100644 --- a/x-pack/plugins/ingest_manager/server/services/agent_policy.ts +++ b/x-pack/plugins/ingest_manager/server/services/agent_policy.ts @@ -83,7 +83,12 @@ class AgentPolicyService { return (await this.get(soClient, id)) as AgentPolicy; } - public async ensureDefaultAgentPolicy(soClient: SavedObjectsClientContract) { + public async ensureDefaultAgentPolicy( + soClient: SavedObjectsClientContract + ): Promise<{ + created: boolean; + defaultAgentPolicy: AgentPolicy; + }> { const agentPolicies = await soClient.find({ type: AGENT_POLICY_SAVED_OBJECT_TYPE, searchFields: ['is_default'], @@ -95,12 +100,18 @@ class AgentPolicyService { ...DEFAULT_AGENT_POLICY, }; - return this.create(soClient, newDefaultAgentPolicy); + return { + created: true, + defaultAgentPolicy: await this.create(soClient, newDefaultAgentPolicy), + }; } return { - id: agentPolicies.saved_objects[0].id, - ...agentPolicies.saved_objects[0].attributes, + created: false, + defaultAgentPolicy: { + id: agentPolicies.saved_objects[0].id, + ...agentPolicies.saved_objects[0].attributes, + }, }; } @@ -404,7 +415,9 @@ class AgentPolicyService { throw new Error('Agent policy not found'); } - const { id: defaultAgentPolicyId } = await this.ensureDefaultAgentPolicy(soClient); + const { + defaultAgentPolicy: { id: defaultAgentPolicyId }, + } = await this.ensureDefaultAgentPolicy(soClient); if (id === defaultAgentPolicyId) { throw new Error('The default agent policy cannot be deleted'); } diff --git a/x-pack/plugins/ingest_manager/server/services/setup.ts b/x-pack/plugins/ingest_manager/server/services/setup.ts index 7f379d3ea4f13d..741a23824f010f 100644 --- a/x-pack/plugins/ingest_manager/server/services/setup.ts +++ b/x-pack/plugins/ingest_manager/server/services/setup.ts @@ -49,7 +49,11 @@ async function createSetupSideEffects( soClient: SavedObjectsClientContract, callCluster: CallESAsCurrentUser ): Promise { - const [installedPackages, defaultOutput, defaultAgentPolicy] = await Promise.all([ + const [ + installedPackages, + defaultOutput, + { created: defaultAgentPolicyCreated, defaultAgentPolicy }, + ] = await Promise.all([ // packages installed by default ensureInstalledDefaultPackages(soClient, callCluster), outputService.ensureDefaultOutput(soClient), @@ -66,44 +70,46 @@ async function createSetupSideEffects( }), ]); - // ensure default packages are added to the default conifg - const agentPolicyWithPackagePolicies = await agentPolicyService.get( - soClient, - defaultAgentPolicy.id, - true - ); - if (!agentPolicyWithPackagePolicies) { - throw new Error('Policy not found'); - } - if ( - agentPolicyWithPackagePolicies.package_policies.length && - typeof agentPolicyWithPackagePolicies.package_policies[0] === 'string' - ) { - throw new Error('Policy not found'); - } - - for (const installedPackage of installedPackages) { - const packageShouldBeInstalled = DEFAULT_AGENT_POLICIES_PACKAGES.some( - (packageName) => installedPackage.name === packageName + // If we just created the default policy, ensure default packages are added to it + if (defaultAgentPolicyCreated) { + const agentPolicyWithPackagePolicies = await agentPolicyService.get( + soClient, + defaultAgentPolicy.id, + true ); - if (!packageShouldBeInstalled) { - continue; + if (!agentPolicyWithPackagePolicies) { + throw new Error('Policy not found'); + } + if ( + agentPolicyWithPackagePolicies.package_policies.length && + typeof agentPolicyWithPackagePolicies.package_policies[0] === 'string' + ) { + throw new Error('Policy not found'); } - const isInstalled = agentPolicyWithPackagePolicies.package_policies.some( - (d: PackagePolicy | string) => { - return typeof d !== 'string' && d.package?.name === installedPackage.name; + for (const installedPackage of installedPackages) { + const packageShouldBeInstalled = DEFAULT_AGENT_POLICIES_PACKAGES.some( + (packageName) => installedPackage.name === packageName + ); + if (!packageShouldBeInstalled) { + continue; } - ); - if (!isInstalled) { - await addPackageToAgentPolicy( - soClient, - callCluster, - installedPackage, - agentPolicyWithPackagePolicies, - defaultOutput + const isInstalled = agentPolicyWithPackagePolicies.package_policies.some( + (d: PackagePolicy | string) => { + return typeof d !== 'string' && d.package?.name === installedPackage.name; + } ); + + if (!isInstalled) { + await addPackageToAgentPolicy( + soClient, + callCluster, + installedPackage, + agentPolicyWithPackagePolicies, + defaultOutput + ); + } } } diff --git a/x-pack/plugins/lens/public/app_plugin/mounter.tsx b/x-pack/plugins/lens/public/app_plugin/mounter.tsx index 3943cbc54f0b3f..ac5d145eedd5ba 100644 --- a/x-pack/plugins/lens/public/app_plugin/mounter.tsx +++ b/x-pack/plugins/lens/public/app_plugin/mounter.tsx @@ -97,9 +97,12 @@ export async function mountApp( const redirectTo = (routeProps: RouteComponentProps<{ id?: string }>, savedObjectId?: string) => { if (!savedObjectId) { - routeProps.history.push('/'); + routeProps.history.push({ pathname: '/', search: routeProps.history.location.search }); } else { - routeProps.history.push(`/edit/${savedObjectId}`); + routeProps.history.push({ + pathname: `/edit/${savedObjectId}`, + search: routeProps.history.location.search, + }); } }; diff --git a/x-pack/plugins/lens/server/plugin.tsx b/x-pack/plugins/lens/server/plugin.tsx index b801d30f5ba9b9..a8f9bef92349cc 100644 --- a/x-pack/plugins/lens/server/plugin.tsx +++ b/x-pack/plugins/lens/server/plugin.tsx @@ -29,13 +29,13 @@ export class LensServerPlugin implements Plugin<{}, {}, {}, {}> { private readonly kibanaIndexConfig: Observable<{ kibana: { index: string } }>; private readonly telemetryLogger: Logger; - constructor(initializerContext: PluginInitializerContext) { + constructor(private initializerContext: PluginInitializerContext) { this.kibanaIndexConfig = initializerContext.config.legacy.globalConfig$; this.telemetryLogger = initializerContext.logger.get('usage'); } setup(core: CoreSetup, plugins: PluginSetupContract) { setupSavedObjects(core); - setupRoutes(core); + setupRoutes(core, this.initializerContext.logger.get()); if (plugins.usageCollection && plugins.taskManager) { registerLensUsageCollector( plugins.usageCollection, diff --git a/x-pack/plugins/lens/server/routes/existing_fields.ts b/x-pack/plugins/lens/server/routes/existing_fields.ts index c925517b572dae..d3b2314a199cb0 100644 --- a/x-pack/plugins/lens/server/routes/existing_fields.ts +++ b/x-pack/plugins/lens/server/routes/existing_fields.ts @@ -7,10 +7,14 @@ import Boom from 'boom'; import { schema } from '@kbn/config-schema'; import { ILegacyScopedClusterClient, SavedObject, RequestHandlerContext } from 'src/core/server'; -import { CoreSetup } from 'src/core/server'; +import { CoreSetup, Logger } from 'src/core/server'; import { BASE_API_URL } from '../../common'; import { IndexPatternAttributes, UI_SETTINGS } from '../../../../../src/plugins/data/server'; +export function isBoomError(error: { isBoom?: boolean }): error is Boom { + return error.isBoom === true; +} + /** * The number of docs to sample to determine field empty status. */ @@ -24,7 +28,7 @@ export interface Field { script?: string; } -export async function existingFieldsRoute(setup: CoreSetup) { +export async function existingFieldsRoute(setup: CoreSetup, logger: Logger) { const router = setup.http.createRouter(); router.post( @@ -52,14 +56,17 @@ export async function existingFieldsRoute(setup: CoreSetup) { }), }); } catch (e) { + logger.info( + `Field existence check failed: ${isBoomError(e) ? e.output.payload.message : e.message}` + ); if (e.status === 404) { - return res.notFound(); + return res.notFound({ body: e.message }); } - if (e.isBoom) { + if (isBoomError(e)) { if (e.output.statusCode === 404) { - return res.notFound(); + return res.notFound({ body: e.output.payload.message }); } - return res.internalError(e.output.message); + return res.internalError({ body: e.output.payload.message }); } else { return res.internalError({ body: Boom.internal(e.message || e.name), diff --git a/x-pack/plugins/lens/server/routes/index.ts b/x-pack/plugins/lens/server/routes/index.ts index 8bc04a56e16d5e..01018d8cd7fe53 100644 --- a/x-pack/plugins/lens/server/routes/index.ts +++ b/x-pack/plugins/lens/server/routes/index.ts @@ -4,13 +4,13 @@ * you may not use this file except in compliance with the Elastic License. */ -import { CoreSetup } from 'src/core/server'; +import { CoreSetup, Logger } from 'src/core/server'; import { existingFieldsRoute } from './existing_fields'; import { initFieldsRoute } from './field_stats'; import { initLensUsageRoute } from './telemetry'; -export function setupRoutes(setup: CoreSetup) { - existingFieldsRoute(setup); +export function setupRoutes(setup: CoreSetup, logger: Logger) { + existingFieldsRoute(setup, logger); initFieldsRoute(setup); initLensUsageRoute(setup); } diff --git a/x-pack/plugins/maps/public/actions/layer_actions.ts b/x-pack/plugins/maps/public/actions/layer_actions.ts index 57f83b9533bdaf..c8c9f6ba400410 100644 --- a/x-pack/plugins/maps/public/actions/layer_actions.ts +++ b/x-pack/plugins/maps/public/actions/layer_actions.ts @@ -441,7 +441,7 @@ export function clearMissingStyleProperties(layerId: string) { const { hasChanges, nextStyleDescriptor, - } = (style as IVectorStyle).getDescriptorWithMissingStylePropsRemoved( + } = await (style as IVectorStyle).getDescriptorWithMissingStylePropsRemoved( nextFields, getMapColors(getState()) ); diff --git a/x-pack/plugins/maps/public/actions/map_actions.ts b/x-pack/plugins/maps/public/actions/map_actions.ts index 4e76bb24c9e34a..f26b00cec9fabb 100644 --- a/x-pack/plugins/maps/public/actions/map_actions.ts +++ b/x-pack/plugins/maps/public/actions/map_actions.ts @@ -282,12 +282,19 @@ export function setRefreshConfig({ isPaused, interval }: MapRefreshConfig) { } export function triggerRefreshTimer() { - return async (dispatch: ThunkDispatch) => { + return async ( + dispatch: ThunkDispatch, + getState: () => MapStoreState + ) => { dispatch({ type: TRIGGER_REFRESH_TIMER, }); - await dispatch(syncDataForAllLayers()); + if (getMapSettings(getState()).autoFitToDataBounds) { + dispatch(autoFitToBounds()); + } else { + await dispatch(syncDataForAllLayers()); + } }; } diff --git a/x-pack/plugins/maps/public/classes/styles/vector/components/vector_style_editor.js b/x-pack/plugins/maps/public/classes/styles/vector/components/vector_style_editor.js index 53a3fc95adbebe..d577912efb830c 100644 --- a/x-pack/plugins/maps/public/classes/styles/vector/components/vector_style_editor.js +++ b/x-pack/plugins/maps/public/classes/styles/vector/components/vector_style_editor.js @@ -20,20 +20,16 @@ import { i18n } from '@kbn/i18n'; import { EuiSpacer, EuiButtonGroup, EuiFormRow, EuiSwitch } from '@elastic/eui'; import { - CATEGORICAL_DATA_TYPES, - ORDINAL_DATA_TYPES, LABEL_BORDER_SIZES, VECTOR_STYLES, STYLE_TYPE, VECTOR_SHAPE_TYPE, } from '../../../../../common/constants'; +import { createStyleFieldsHelper } from '../style_fields_helper'; export class VectorStyleEditor extends Component { state = { - dateFields: [], - numberFields: [], - fields: [], - ordinalAndCategoricalFields: [], + styleFields: [], defaultDynamicProperties: getDefaultDynamicProperties(), defaultStaticProperties: getDefaultStaticProperties(), supportedFeatures: undefined, @@ -56,33 +52,17 @@ export class VectorStyleEditor extends Component { } async _loadFields() { - const getFieldMeta = async (field) => { - return { - label: await field.getLabel(), - name: field.getName(), - origin: field.getOrigin(), - type: await field.getDataType(), - supportsAutoDomain: field.supportsAutoDomain(), - }; - }; - - //These are all fields (only used for text labeling) - const fields = await this.props.layer.getStyleEditorFields(); - const fieldPromises = fields.map(getFieldMeta); - const fieldsArrayAll = await Promise.all(fieldPromises); - if (!this._isMounted || _.isEqual(fieldsArrayAll, this.state.fields)) { + const styleFieldsHelper = await createStyleFieldsHelper( + await this.props.layer.getStyleEditorFields() + ); + const styleFields = styleFieldsHelper.getStyleFields(); + if (!this._isMounted || _.isEqual(styleFields, this.state.styleFields)) { return; } this.setState({ - fields: fieldsArrayAll, - ordinalAndCategoricalFields: fieldsArrayAll.filter((field) => { - return ( - CATEGORICAL_DATA_TYPES.includes(field.type) || ORDINAL_DATA_TYPES.includes(field.type) - ); - }), - dateFields: fieldsArrayAll.filter((field) => field.type === 'date'), - numberFields: fieldsArrayAll.filter((field) => field.type === 'number'), + styleFields, + styleFieldsHelper, }); } @@ -109,12 +89,6 @@ export class VectorStyleEditor extends Component { } } - _getOrdinalFields() { - return [...this.state.dateFields, ...this.state.numberFields].filter((field) => { - return field.supportsAutoDomain; - }); - } - _handleSelectedFeatureChange = (selectedFeature) => { this.setState({ selectedFeature }); }; @@ -165,7 +139,7 @@ export class VectorStyleEditor extends Component { onStaticStyleChange={this._onStaticStyleChange} onDynamicStyleChange={this._onDynamicStyleChange} styleProperty={this.props.styleProperties[VECTOR_STYLES.FILL_COLOR]} - fields={this.state.ordinalAndCategoricalFields} + fields={this.state.styleFieldsHelper.getFieldsForStyle(VECTOR_STYLES.FILL_COLOR)} defaultStaticStyleOptions={ this.state.defaultStaticProperties[VECTOR_STYLES.FILL_COLOR].options } @@ -186,7 +160,7 @@ export class VectorStyleEditor extends Component { onStaticStyleChange={this._onStaticStyleChange} onDynamicStyleChange={this._onDynamicStyleChange} styleProperty={this.props.styleProperties[VECTOR_STYLES.LINE_COLOR]} - fields={this.state.ordinalAndCategoricalFields} + fields={this.state.styleFieldsHelper.getFieldsForStyle(VECTOR_STYLES.LINE_COLOR)} defaultStaticStyleOptions={ this.state.defaultStaticProperties[VECTOR_STYLES.LINE_COLOR].options } @@ -205,7 +179,7 @@ export class VectorStyleEditor extends Component { onStaticStyleChange={this._onStaticStyleChange} onDynamicStyleChange={this._onDynamicStyleChange} styleProperty={this.props.styleProperties[VECTOR_STYLES.LINE_WIDTH]} - fields={this._getOrdinalFields()} + fields={this.state.styleFieldsHelper.getFieldsForStyle(VECTOR_STYLES.LINE_WIDTH)} defaultStaticStyleOptions={ this.state.defaultStaticProperties[VECTOR_STYLES.LINE_WIDTH].options } @@ -225,7 +199,7 @@ export class VectorStyleEditor extends Component { onStaticStyleChange={this._onStaticStyleChange} onDynamicStyleChange={this._onDynamicStyleChange} styleProperty={this.props.styleProperties[VECTOR_STYLES.LABEL_TEXT]} - fields={this.state.fields} + fields={this.state.styleFieldsHelper.getFieldsForStyle(VECTOR_STYLES.LABEL_TEXT)} defaultStaticStyleOptions={ this.state.defaultStaticProperties[VECTOR_STYLES.LABEL_TEXT].options } @@ -242,7 +216,7 @@ export class VectorStyleEditor extends Component { onStaticStyleChange={this._onStaticStyleChange} onDynamicStyleChange={this._onDynamicStyleChange} styleProperty={this.props.styleProperties[VECTOR_STYLES.LABEL_COLOR]} - fields={this.state.ordinalAndCategoricalFields} + fields={this.state.styleFieldsHelper.getFieldsForStyle(VECTOR_STYLES.LABEL_COLOR)} defaultStaticStyleOptions={ this.state.defaultStaticProperties[VECTOR_STYLES.LABEL_COLOR].options } @@ -258,7 +232,7 @@ export class VectorStyleEditor extends Component { onStaticStyleChange={this._onStaticStyleChange} onDynamicStyleChange={this._onDynamicStyleChange} styleProperty={this.props.styleProperties[VECTOR_STYLES.LABEL_SIZE]} - fields={this._getOrdinalFields()} + fields={this.state.styleFieldsHelper.getFieldsForStyle(VECTOR_STYLES.LABEL_SIZE)} defaultStaticStyleOptions={ this.state.defaultStaticProperties[VECTOR_STYLES.LABEL_SIZE].options } @@ -275,7 +249,7 @@ export class VectorStyleEditor extends Component { onStaticStyleChange={this._onStaticStyleChange} onDynamicStyleChange={this._onDynamicStyleChange} styleProperty={this.props.styleProperties[VECTOR_STYLES.LABEL_BORDER_COLOR]} - fields={this.state.ordinalAndCategoricalFields} + fields={this.state.styleFieldsHelper.getFieldsForStyle(VECTOR_STYLES.LABEL_BORDER_COLOR)} defaultStaticStyleOptions={ this.state.defaultStaticProperties[VECTOR_STYLES.LABEL_BORDER_COLOR].options } @@ -309,7 +283,7 @@ export class VectorStyleEditor extends Component { onStaticStyleChange={this._onStaticStyleChange} onDynamicStyleChange={this._onDynamicStyleChange} styleProperty={this.props.styleProperties[VECTOR_STYLES.ICON_ORIENTATION]} - fields={this.state.numberFields} + fields={this.state.styleFieldsHelper.getFieldsForStyle(VECTOR_STYLES.ICON_ORIENTATION)} defaultStaticStyleOptions={ this.state.defaultStaticProperties[VECTOR_STYLES.ICON_ORIENTATION].options } @@ -328,7 +302,7 @@ export class VectorStyleEditor extends Component { onStaticStyleChange={this._onStaticStyleChange} onDynamicStyleChange={this._onDynamicStyleChange} styleProperty={this.props.styleProperties[VECTOR_STYLES.ICON]} - fields={this.state.ordinalAndCategoricalFields} + fields={this.state.styleFieldsHelper.getFieldsForStyle(VECTOR_STYLES.ICON)} defaultStaticStyleOptions={ this.state.defaultStaticProperties[VECTOR_STYLES.ICON].options } @@ -368,7 +342,7 @@ export class VectorStyleEditor extends Component { onStaticStyleChange={this._onStaticStyleChange} onDynamicStyleChange={this._onDynamicStyleChange} styleProperty={this.props.styleProperties[VECTOR_STYLES.ICON_SIZE]} - fields={this._getOrdinalFields()} + fields={this.state.styleFieldsHelper.getFieldsForStyle(VECTOR_STYLES.ICON_SIZE)} defaultStaticStyleOptions={ this.state.defaultStaticProperties[VECTOR_STYLES.ICON_SIZE].options } @@ -409,9 +383,9 @@ export class VectorStyleEditor extends Component { } _renderProperties() { - const { supportedFeatures, selectedFeature } = this.state; + const { supportedFeatures, selectedFeature, styleFieldsHelper } = this.state; - if (!supportedFeatures) { + if (!supportedFeatures || !styleFieldsHelper) { return null; } diff --git a/x-pack/plugins/maps/public/classes/styles/vector/properties/__tests__/test_util.ts b/x-pack/plugins/maps/public/classes/styles/vector/properties/__tests__/test_util.ts index a2dfdc94d8058c..a3f63ba7717ea6 100644 --- a/x-pack/plugins/maps/public/classes/styles/vector/properties/__tests__/test_util.ts +++ b/x-pack/plugins/maps/public/classes/styles/vector/properties/__tests__/test_util.ts @@ -16,10 +16,38 @@ import { import { AbstractField, IField } from '../../../../fields/field'; import { IStyle } from '../../../style'; -class MockField extends AbstractField { +export class MockField extends AbstractField { + private readonly _dataType: string; + private readonly _supportsAutoDomain: boolean; + + constructor({ + fieldName, + origin = FIELD_ORIGIN.SOURCE, + dataType = 'string', + supportsAutoDomain = true, + }: { + fieldName: string; + origin?: FIELD_ORIGIN; + dataType?: string; + supportsAutoDomain?: boolean; + }) { + super({ fieldName, origin }); + this._dataType = dataType; + this._supportsAutoDomain = supportsAutoDomain; + } + async getLabel(): Promise { return this.getName() + '_label'; } + + async getDataType(): Promise { + return this._dataType; + } + + supportsAutoDomain(): boolean { + return this._supportsAutoDomain; + } + supportsFieldMeta(): boolean { return true; } diff --git a/x-pack/plugins/maps/public/classes/styles/vector/style_fields_helper.ts b/x-pack/plugins/maps/public/classes/styles/vector/style_fields_helper.ts new file mode 100644 index 00000000000000..8613f9e1e946ff --- /dev/null +++ b/x-pack/plugins/maps/public/classes/styles/vector/style_fields_helper.ts @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { + CATEGORICAL_DATA_TYPES, + FIELD_ORIGIN, + ORDINAL_DATA_TYPES, + VECTOR_STYLES, +} from '../../../../common/constants'; +import { IField } from '../../fields/field'; + +export interface StyleField { + label: string; + name: string; + origin: FIELD_ORIGIN; + type: string; + supportsAutoDomain: boolean; +} + +export async function createStyleFieldsHelper(fields: IField[]): Promise { + const promises: Array> = fields.map(async (field: IField) => { + return { + label: await field.getLabel(), + name: field.getName(), + origin: field.getOrigin(), + type: await field.getDataType(), + supportsAutoDomain: field.supportsAutoDomain(), + }; + }); + const styleFields = await Promise.all(promises); + return new StyleFieldsHelper(styleFields); +} + +class StyleFieldsHelper { + private readonly _styleFields: StyleField[]; + private readonly _ordinalAndCategoricalFields: StyleField[]; + private readonly _numberFields: StyleField[]; + private readonly _ordinalFields: StyleField[]; + + constructor(styleFields: StyleField[]) { + const ordinalAndCategoricalFields: StyleField[] = []; + const numberFields: StyleField[] = []; + const ordinalFields: StyleField[] = []; + + styleFields.forEach((styleField: StyleField) => { + if ( + CATEGORICAL_DATA_TYPES.includes(styleField.type) || + ORDINAL_DATA_TYPES.includes(styleField.type) + ) { + ordinalAndCategoricalFields.push(styleField); + } + + if (styleField.type === 'date' || styleField.type === 'number') { + if (styleField.type === 'number') { + numberFields.push(styleField); + } + if (styleField.supportsAutoDomain) { + ordinalFields.push(styleField); + } + } + }); + + this._styleFields = styleFields; + this._ordinalAndCategoricalFields = ordinalAndCategoricalFields; + this._numberFields = numberFields; + this._ordinalFields = ordinalFields; + } + + getFieldsForStyle(styleName: VECTOR_STYLES): StyleField[] { + switch (styleName) { + case VECTOR_STYLES.ICON_ORIENTATION: + return this._numberFields; + case VECTOR_STYLES.FILL_COLOR: + case VECTOR_STYLES.LINE_COLOR: + case VECTOR_STYLES.LABEL_COLOR: + case VECTOR_STYLES.LABEL_BORDER_COLOR: + case VECTOR_STYLES.ICON: + return this._ordinalAndCategoricalFields; + case VECTOR_STYLES.LINE_WIDTH: + case VECTOR_STYLES.LABEL_SIZE: + case VECTOR_STYLES.ICON_SIZE: + return this._ordinalFields; + case VECTOR_STYLES.LABEL_TEXT: + return this._styleFields; + default: + return []; + } + } + + getStyleFields(): StyleField[] { + return this._styleFields; + } +} diff --git a/x-pack/plugins/maps/public/classes/styles/vector/vector_style.test.js b/x-pack/plugins/maps/public/classes/styles/vector/vector_style.test.js index 28801a402ca14d..1dbadc054c8a06 100644 --- a/x-pack/plugins/maps/public/classes/styles/vector/vector_style.test.js +++ b/x-pack/plugins/maps/public/classes/styles/vector/vector_style.test.js @@ -12,23 +12,10 @@ import { VECTOR_SHAPE_TYPE, VECTOR_STYLES, } from '../../../../common/constants'; +import { MockField } from './properties/__tests__/test_util'; jest.mock('../../../kibana_services'); -class MockField { - constructor({ fieldName }) { - this._fieldName = fieldName; - } - - getName() { - return this._fieldName; - } - - isValid() { - return !!this._fieldName; - } -} - class MockSource { constructor({ supportedShapeTypes } = {}) { this._supportedShapeTypes = supportedShapeTypes || Object.values(VECTOR_SHAPE_TYPE); @@ -77,25 +64,25 @@ describe('getDescriptorWithMissingStylePropsRemoved', () => { }); }); - it('Should return no changes when next ordinal fields contain existing style property fields', () => { + it('Should return no changes when next ordinal fields contain existing style property fields', async () => { const vectorStyle = new VectorStyle({ properties }, new MockSource()); - const nextFields = [new MockField({ fieldName })]; - const { hasChanges } = vectorStyle.getDescriptorWithMissingStylePropsRemoved( + const nextFields = [new MockField({ fieldName, dataType: 'number' })]; + const { hasChanges } = await vectorStyle.getDescriptorWithMissingStylePropsRemoved( nextFields, mapColors ); expect(hasChanges).toBe(false); }); - it('Should clear missing fields when next ordinal fields do not contain existing style property fields', () => { + it('Should clear missing fields when next ordinal fields do not contain existing style property fields', async () => { const vectorStyle = new VectorStyle({ properties }, new MockSource()); - const nextFields = [new MockField({ fieldName: 'someOtherField' })]; + const nextFields = [new MockField({ fieldName: 'someOtherField', dataType: 'number' })]; const { hasChanges, nextStyleDescriptor, - } = vectorStyle.getDescriptorWithMissingStylePropsRemoved(nextFields, mapColors); + } = await vectorStyle.getDescriptorWithMissingStylePropsRemoved(nextFields, mapColors); expect(hasChanges).toBe(true); expect(nextStyleDescriptor.properties[VECTOR_STYLES.LINE_COLOR]).toEqual({ options: {}, @@ -110,14 +97,14 @@ describe('getDescriptorWithMissingStylePropsRemoved', () => { }); }); - it('Should convert dynamic styles to static styles when there are no next fields', () => { + it('Should convert dynamic styles to static styles when there are no next fields', async () => { const vectorStyle = new VectorStyle({ properties }, new MockSource()); const nextFields = []; const { hasChanges, nextStyleDescriptor, - } = vectorStyle.getDescriptorWithMissingStylePropsRemoved(nextFields, mapColors); + } = await vectorStyle.getDescriptorWithMissingStylePropsRemoved(nextFields, mapColors); expect(hasChanges).toBe(true); expect(nextStyleDescriptor.properties[VECTOR_STYLES.LINE_COLOR]).toEqual({ options: { @@ -132,6 +119,25 @@ describe('getDescriptorWithMissingStylePropsRemoved', () => { type: 'STATIC', }); }); + + it('Should convert dynamic ICON_SIZE static style when there are no next ordinal fields', async () => { + const vectorStyle = new VectorStyle({ properties }, new MockSource()); + + const nextFields = [ + new MockField({ fieldName, dataType: 'number', supportsAutoDomain: false }), + ]; + const { + hasChanges, + nextStyleDescriptor, + } = await vectorStyle.getDescriptorWithMissingStylePropsRemoved(nextFields, mapColors); + expect(hasChanges).toBe(true); + expect(nextStyleDescriptor.properties[VECTOR_STYLES.ICON_SIZE]).toEqual({ + options: { + size: 6, + }, + type: 'STATIC', + }); + }); }); describe('pluckStyleMetaFromSourceDataRequest', () => { diff --git a/x-pack/plugins/maps/public/classes/styles/vector/vector_style.tsx b/x-pack/plugins/maps/public/classes/styles/vector/vector_style.tsx index acb158636e0b36..5d360f5452d36e 100644 --- a/x-pack/plugins/maps/public/classes/styles/vector/vector_style.tsx +++ b/x-pack/plugins/maps/public/classes/styles/vector/vector_style.tsx @@ -77,6 +77,7 @@ import { IField } from '../../fields/field'; import { IVectorLayer } from '../../layers/vector_layer/vector_layer'; import { IVectorSource } from '../../sources/vector_source'; import { ILayer } from '../../layers/layer'; +import { createStyleFieldsHelper } from './style_fields_helper'; const POINTS = [GEO_JSON_TYPE.POINT, GEO_JSON_TYPE.MULTI_POINT]; const LINES = [GEO_JSON_TYPE.LINE_STRING, GEO_JSON_TYPE.MULTI_LINE_STRING]; @@ -90,7 +91,7 @@ export interface IVectorStyle extends IStyle { getDescriptorWithMissingStylePropsRemoved( nextFields: IField[], mapColors: string[] - ): { hasChanges: boolean; nextStyleDescriptor?: VectorStyleDescriptor }; + ): Promise<{ hasChanges: boolean; nextStyleDescriptor?: VectorStyleDescriptor }>; pluckStyleMetaFromSourceDataRequest(sourceDataRequest: DataRequest): Promise; isTimeAware: () => boolean; getIcon: () => ReactElement; @@ -324,7 +325,8 @@ export class VectorStyle implements IVectorStyle { * This method does not update its descriptor. It just returns a new descriptor that the caller * can then use to update store state via dispatch. */ - getDescriptorWithMissingStylePropsRemoved(nextFields: IField[], mapColors: string[]) { + async getDescriptorWithMissingStylePropsRemoved(nextFields: IField[], mapColors: string[]) { + const styleFieldsHelper = await createStyleFieldsHelper(nextFields); const originalProperties = this.getRawProperties(); const updatedProperties = {} as VectorStylePropertiesDescriptor; @@ -346,8 +348,9 @@ export class VectorStyle implements IVectorStyle { }); dynamicProperties.forEach((key: VECTOR_STYLES) => { - // Convert dynamic styling to static stying when there are no nextFields - if (nextFields.length === 0) { + // Convert dynamic styling to static stying when there are no style fields + const styleFields = styleFieldsHelper.getFieldsForStyle(key); + if (styleFields.length === 0) { const staticProperties = getDefaultStaticProperties(mapColors); updatedProperties[key] = staticProperties[key] as any; return; diff --git a/x-pack/plugins/maps/public/connected_components/map_settings_panel/__snapshots__/navigation_panel.test.tsx.snap b/x-pack/plugins/maps/public/connected_components/map_settings_panel/__snapshots__/navigation_panel.test.tsx.snap index a617fbc552854d..1859c7d8177f8f 100644 --- a/x-pack/plugins/maps/public/connected_components/map_settings_panel/__snapshots__/navigation_panel.test.tsx.snap +++ b/x-pack/plugins/maps/public/connected_components/map_settings_panel/__snapshots__/navigation_panel.test.tsx.snap @@ -25,7 +25,7 @@ exports[`should render 1`] = ` labelType="label" > { + onClick: async () => { if (getHasUnsavedChanges()) { - const navigateAway = window.confirm(unsavedChangesWarning); - if (navigateAway) { + const confirmed = await getCoreOverlays().openConfirm(unsavedChangesWarning, { + title: unsavedChangesTitle, + 'data-test-subj': 'appLeaveConfirmModal', + }); + if (confirmed) { goToSpecifiedPath('/'); } } else { diff --git a/x-pack/plugins/maps/public/routing/routes/maps_app/maps_app_view.tsx b/x-pack/plugins/maps/public/routing/routes/maps_app/maps_app_view.tsx index bd08b2f11fadcf..df46d5d6a13ff6 100644 --- a/x-pack/plugins/maps/public/routing/routes/maps_app/maps_app_view.tsx +++ b/x-pack/plugins/maps/public/routing/routes/maps_app/maps_app_view.tsx @@ -43,7 +43,7 @@ import { import { MapContainer } from '../../../connected_components/map_container'; import { getIndexPatternsFromIds } from '../../../index_pattern_util'; import { getTopNavConfig } from './top_nav_config'; -import { getBreadcrumbs, unsavedChangesWarning } from './get_breadcrumbs'; +import { getBreadcrumbs, unsavedChangesTitle, unsavedChangesWarning } from './get_breadcrumbs'; import { LayerDescriptor, MapRefreshConfig, @@ -138,9 +138,7 @@ export class MapsAppView extends React.Component { this.props.onAppLeave((actions) => { if (this._hasUnsavedChanges()) { - if (!window.confirm(unsavedChangesWarning)) { - return {} as AppLeaveAction; - } + return actions.confirm(unsavedChangesWarning, unsavedChangesTitle); } return actions.default() as AppLeaveAction; }); diff --git a/x-pack/plugins/maps/server/routes.js b/x-pack/plugins/maps/server/routes.js index 6b19103b597228..5feacfee4d4d29 100644 --- a/x-pack/plugins/maps/server/routes.js +++ b/x-pack/plugins/maps/server/routes.js @@ -189,18 +189,28 @@ export function initRoutes(router, licenseUid, mapConfig, kbnVersion, logger) { return badRequest('map.proxyElasticMapsServiceInMaps disabled'); } - const file = await emsClient.getDefaultFileManifest(); - const layers = file.layers.map((layer) => { - const newLayer = { ...layer }; - const id = encodeURIComponent(layer.layer_id); + const file = await emsClient.getDefaultFileManifest(); //need raw manifest + const fileLayers = await emsClient.getFileLayers(); + + const layers = file.layers.map((layerJson) => { + const newLayerJson = { ...layerJson }; + const id = encodeURIComponent(layerJson.layer_id); + + const fileLayer = fileLayers.find((fileLayer) => fileLayer.getId() === layerJson.layer_id); + const defaultFormat = layerJson.formats.find( + (format) => format.type === fileLayer.getDefaultFormatType() + ); + const newUrl = `${EMS_FILES_DEFAULT_JSON_PATH}?id=${id}`; - newLayer.formats = [ + + //Only proxy default-format. Others are unused in Maps-app + newLayerJson.formats = [ { - ...layer.formats[0], + ...defaultFormat, url: newUrl, }, ]; - return newLayer; + return newLayerJson; }); //rewrite return ok({ diff --git a/x-pack/plugins/ml/public/application/timeseriesexplorer/_timeseriesexplorer.scss b/x-pack/plugins/ml/public/application/timeseriesexplorer/_timeseriesexplorer.scss index 8d4f6ad0a6844a..45678f6e71c2ea 100644 --- a/x-pack/plugins/ml/public/application/timeseriesexplorer/_timeseriesexplorer.scss +++ b/x-pack/plugins/ml/public/application/timeseriesexplorer/_timeseriesexplorer.scss @@ -24,10 +24,6 @@ } } - .single-metric-request-callout { - margin: 0 $euiSize; - } - .results-container { .panel-title { font-size: $euiFontSizeM; diff --git a/x-pack/plugins/ml/public/application/timeseriesexplorer/components/entity_control/entity_control.tsx b/x-pack/plugins/ml/public/application/timeseriesexplorer/components/entity_control/entity_control.tsx index 9d2c49a95fec45..e1323019d61dbd 100644 --- a/x-pack/plugins/ml/public/application/timeseriesexplorer/components/entity_control/entity_control.tsx +++ b/x-pack/plugins/ml/public/application/timeseriesexplorer/components/entity_control/entity_control.tsx @@ -9,13 +9,7 @@ import React, { Component } from 'react'; import { FormattedMessage } from '@kbn/i18n/react'; import { i18n } from '@kbn/i18n'; -import { - EuiComboBox, - EuiComboBoxOptionOption, - EuiFlexItem, - EuiFormRow, - EuiToolTip, -} from '@elastic/eui'; +import { EuiComboBox, EuiComboBoxOptionOption, EuiFlexItem, EuiFormRow } from '@elastic/eui'; export interface Entity { fieldName: string; @@ -156,9 +150,7 @@ export class EntityControl extends Component - - {control} - + {control} ); diff --git a/x-pack/plugins/ml/public/application/timeseriesexplorer/timeseriesexplorer.js b/x-pack/plugins/ml/public/application/timeseriesexplorer/timeseriesexplorer.js index 7d173e161a1cb3..9b8764d3f92796 100644 --- a/x-pack/plugins/ml/public/application/timeseriesexplorer/timeseriesexplorer.js +++ b/x-pack/plugins/ml/public/application/timeseriesexplorer/timeseriesexplorer.js @@ -1133,23 +1133,25 @@ export class TimeSeriesExplorer extends React.Component { return ( {fieldNamesWithEmptyValues.length > 0 && ( - - } - color="warning" - iconType="help" - size="s" - /> + <> + + } + color="warning" + iconType="help" + size="s" + /> + + )}
diff --git a/x-pack/plugins/monitoring/public/components/elasticsearch/node/advanced.js b/x-pack/plugins/monitoring/public/components/elasticsearch/node/advanced.js index b2a17515bbb967..67df745e619d40 100644 --- a/x-pack/plugins/monitoring/public/components/elasticsearch/node/advanced.js +++ b/x-pack/plugins/monitoring/public/components/elasticsearch/node/advanced.js @@ -18,8 +18,9 @@ import { import { NodeDetailStatus } from '../node_detail_status'; import { MonitoringTimeseriesContainer } from '../../chart'; import { FormattedMessage } from '@kbn/i18n/react'; +import { AlertsCallout } from '../../../alerts/callout'; -export const AdvancedNode = ({ nodeSummary, metrics, alerts, ...props }) => { +export const AdvancedNode = ({ nodeSummary, metrics, alerts, nodeId, ...props }) => { const metricsToShow = [ metrics.node_gc, metrics.node_gc_time, @@ -50,9 +51,25 @@ export const AdvancedNode = ({ nodeSummary, metrics, alerts, ...props }) => { - + + state.nodeId === nodeId || state.stackProductUuid === nodeId + } + /> + state.nodeId === nodeId || state.stackProductUuid === nodeId} + nextStepsFilter={(nextStep) => { + if (nextStep.text.includes('Elasticsearch nodes')) { + return false; + } + return true; + }} + /> {metricsToShow.map((metric, index) => ( diff --git a/x-pack/plugins/monitoring/public/views/elasticsearch/node/advanced/index.js b/x-pack/plugins/monitoring/public/views/elasticsearch/node/advanced/index.js index 03c0714864f922..8021ae7e5f63c4 100644 --- a/x-pack/plugins/monitoring/public/views/elasticsearch/node/advanced/index.js +++ b/x-pack/plugins/monitoring/public/views/elasticsearch/node/advanced/index.js @@ -117,6 +117,7 @@ uiRoutes.when('/elasticsearch/nodes/:node/advanced', { { - let cpuUsage = 0; if (this.config.ui.container.elasticsearch.enabled) { - cpuUsage = + stat.cpuUsage = (stat.containerUsage / (stat.containerPeriods * stat.containerQuota * 1000)) * 100; - } else { - cpuUsage = stat.cpuUsage; } return { instanceKey: `${stat.clusterUuid}:${stat.nodeId}`, clusterUuid: stat.clusterUuid, - shouldFire: cpuUsage > params.threshold, + shouldFire: stat.cpuUsage > params.threshold, severity: AlertSeverity.Danger, meta: stat, ccs: stat.ccs, diff --git a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_settings_collector.ts b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_settings_collector.ts index a3ff4b952ce979..0dd5ce291f9721 100644 --- a/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_settings_collector.ts +++ b/x-pack/plugins/monitoring/server/kibana_monitoring/collectors/get_settings_collector.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { Collector } from '../../../../../../src/plugins/usage_collection/server'; +import { Collector, UsageCollectionSetup } from 'src/plugins/usage_collection/server'; import { KIBANA_SETTINGS_TYPE } from '../../../common/constants'; import { MonitoringConfig } from '../../config'; @@ -48,10 +48,18 @@ export interface KibanaSettingsCollector extends Collector true, + schema: { + xpack: { + default_admin_email: { type: 'text' }, + }, + }, async fetch(this: KibanaSettingsCollector) { let kibanaSettingsData; const defaultAdminEmail = await checkForEmailValue(config); diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_cpu_usage_node_stats.test.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_cpu_usage_node_stats.test.ts index 12926a30efa1bc..88035c11218488 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_cpu_usage_node_stats.test.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_cpu_usage_node_stats.test.ts @@ -97,11 +97,18 @@ describe('fetchCpuUsageNodeStats', () => { }, ], }, - average_usage: { - value: 10, - }, - average_periods: { - value: 5, + histo: { + buckets: [ + null, + { + usage_deriv: { + normalized_value: 10, + }, + periods_deriv: { + normalized_value: 5, + }, + }, + ], }, average_quota: { value: 50, @@ -185,14 +192,14 @@ describe('fetchCpuUsageNodeStats', () => { }); await fetchCpuUsageNodeStats(callCluster, clusters, index, startMs, endMs, size); expect(params).toStrictEqual({ - index, + index: '.monitoring-es-*', filterPath: ['aggregations'], body: { size: 0, query: { bool: { filter: [ - { terms: { cluster_uuid: clusters.map((cluster) => cluster.clusterUuid) } }, + { terms: { cluster_uuid: ['abc123'] } }, { term: { type: 'node_stats' } }, { range: { timestamp: { format: 'epoch_millis', gte: 0, lte: 0 } } }, ], @@ -200,23 +207,38 @@ describe('fetchCpuUsageNodeStats', () => { }, aggs: { clusters: { - terms: { - field: 'cluster_uuid', - size, - include: clusters.map((cluster) => cluster.clusterUuid), - }, + terms: { field: 'cluster_uuid', size: 10, include: ['abc123'] }, aggs: { nodes: { - terms: { field: 'node_stats.node_id', size }, + terms: { field: 'node_stats.node_id', size: 10 }, aggs: { index: { terms: { field: '_index', size: 1 } }, average_cpu: { avg: { field: 'node_stats.process.cpu.percent' } }, - average_usage: { avg: { field: 'node_stats.os.cgroup.cpuacct.usage_nanos' } }, - average_periods: { - avg: { field: 'node_stats.os.cgroup.cpu.stat.number_of_elapsed_periods' }, - }, average_quota: { avg: { field: 'node_stats.os.cgroup.cpu.cfs_quota_micros' } }, name: { terms: { field: 'source_node.name', size: 1 } }, + histo: { + date_histogram: { field: 'timestamp', fixed_interval: '0m' }, + aggs: { + average_periods: { + max: { field: 'node_stats.os.cgroup.cpu.stat.number_of_elapsed_periods' }, + }, + average_usage: { max: { field: 'node_stats.os.cgroup.cpuacct.usage_nanos' } }, + usage_deriv: { + derivative: { + buckets_path: 'average_usage', + gap_policy: 'skip', + unit: '1s', + }, + }, + periods_deriv: { + derivative: { + buckets_path: 'average_periods', + gap_policy: 'skip', + unit: '1s', + }, + }, + }, + }, }, }, }, diff --git a/x-pack/plugins/monitoring/server/lib/alerts/fetch_cpu_usage_node_stats.ts b/x-pack/plugins/monitoring/server/lib/alerts/fetch_cpu_usage_node_stats.ts index 4fdb03b61950e9..ecd324c083a8c4 100644 --- a/x-pack/plugins/monitoring/server/lib/alerts/fetch_cpu_usage_node_stats.ts +++ b/x-pack/plugins/monitoring/server/lib/alerts/fetch_cpu_usage_node_stats.ts @@ -4,6 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ import { get } from 'lodash'; +import moment from 'moment'; +import { NORMALIZED_DERIVATIVE_UNIT } from '../../../common/constants'; import { AlertCluster, AlertCpuUsageNodeStats } from '../../alerts/types'; interface NodeBucketESResponse { @@ -26,6 +28,9 @@ export async function fetchCpuUsageNodeStats( endMs: number, size: number ): Promise { + // Using pure MS didn't seem to work well with the date_histogram interval + // but minutes does + const intervalInMinutes = moment.duration(endMs - startMs).asMinutes(); const filterPath = ['aggregations']; const params = { index, @@ -82,16 +87,6 @@ export async function fetchCpuUsageNodeStats( field: 'node_stats.process.cpu.percent', }, }, - average_usage: { - avg: { - field: 'node_stats.os.cgroup.cpuacct.usage_nanos', - }, - }, - average_periods: { - avg: { - field: 'node_stats.os.cgroup.cpu.stat.number_of_elapsed_periods', - }, - }, average_quota: { avg: { field: 'node_stats.os.cgroup.cpu.cfs_quota_micros', @@ -103,6 +98,38 @@ export async function fetchCpuUsageNodeStats( size: 1, }, }, + histo: { + date_histogram: { + field: 'timestamp', + fixed_interval: `${intervalInMinutes}m`, + }, + aggs: { + average_periods: { + max: { + field: 'node_stats.os.cgroup.cpu.stat.number_of_elapsed_periods', + }, + }, + average_usage: { + max: { + field: 'node_stats.os.cgroup.cpuacct.usage_nanos', + }, + }, + usage_deriv: { + derivative: { + buckets_path: 'average_usage', + gap_policy: 'skip', + unit: NORMALIZED_DERIVATIVE_UNIT, + }, + }, + periods_deriv: { + derivative: { + buckets_path: 'average_periods', + gap_policy: 'skip', + unit: NORMALIZED_DERIVATIVE_UNIT, + }, + }, + }, + }, }, }, }, @@ -120,17 +147,19 @@ export async function fetchCpuUsageNodeStats( ) as ClusterBucketESResponse[]; for (const clusterBucket of clusterBuckets) { for (const node of clusterBucket.nodes.buckets) { + const lastBucket = get(node, 'histo.buckets[1]', {}); const indexName = get(node, 'index.buckets[0].key', ''); - stats.push({ + const stat = { clusterUuid: clusterBucket.key, nodeId: node.key, nodeName: get(node, 'name.buckets[0].key'), cpuUsage: get(node, 'average_cpu.value'), - containerUsage: get(node, 'average_usage.value'), - containerPeriods: get(node, 'average_periods.value'), + containerUsage: get(lastBucket, 'usage_deriv.normalized_value'), + containerPeriods: get(lastBucket, 'periods_deriv.normalized_value'), containerQuota: get(node, 'average_quota.value'), ccs: indexName.includes(':') ? indexName.split(':')[0] : null, - }); + }; + stats.push(stat); } } return stats; diff --git a/x-pack/plugins/reporting/public/panel_actions/get_csv_panel_action.tsx b/x-pack/plugins/reporting/public/panel_actions/get_csv_panel_action.tsx index 7782576bb5a00c..1e3f7e34bebdb1 100644 --- a/x-pack/plugins/reporting/public/panel_actions/get_csv_panel_action.tsx +++ b/x-pack/plugins/reporting/public/panel_actions/get_csv_panel_action.tsx @@ -108,7 +108,7 @@ export class GetCsvReportPanelAction implements ActionDefinition const filename = embeddable.getSavedSearch().title; const timezone = kibanaTimezone === 'Browser' ? moment.tz.guess() : kibanaTimezone; const fromTime = dateMath.parse(from); - const toTime = dateMath.parse(to); + const toTime = dateMath.parse(to, { roundUp: true }); if (!fromTime || !toTime) { return this.onGenerationFail( diff --git a/x-pack/plugins/rollup/server/routes/api/index_patterns/register_fields_for_wildcard_route.ts b/x-pack/plugins/rollup/server/routes/api/index_patterns/register_fields_for_wildcard_route.ts index 250947d72c5fab..df9907fbf731a1 100644 --- a/x-pack/plugins/rollup/server/routes/api/index_patterns/register_fields_for_wildcard_route.ts +++ b/x-pack/plugins/rollup/server/routes/api/index_patterns/register_fields_for_wildcard_route.ts @@ -8,6 +8,7 @@ import { keyBy } from 'lodash'; import { schema } from '@kbn/config-schema'; import { Field } from '../../../lib/merge_capabilities_with_fields'; import { RouteDependencies } from '../../../types'; +import type { IndexPatternsFetcher as IndexPatternsFetcherType } from '../../../../../../../src/plugins/data/server'; const parseMetaFields = (metaFields: string | string[]) => { let parsedFields: string[] = []; @@ -23,10 +24,10 @@ const getFieldsForWildcardRequest = async ( context: any, request: any, response: any, - IndexPatternsFetcher: any + IndexPatternsFetcher: typeof IndexPatternsFetcherType ) => { - const { callAsCurrentUser } = context.core.elasticsearch.legacy.client; - const indexPatterns = new IndexPatternsFetcher(callAsCurrentUser); + const { asCurrentUser } = context.core.elasticsearch.client; + const indexPatterns = new IndexPatternsFetcher(asCurrentUser); const { pattern, meta_fields: metaFields } = request.query; let parsedFields: string[] = []; diff --git a/x-pack/plugins/security/public/management/role_mappings/role_mappings_grid/role_mappings_grid_page.tsx b/x-pack/plugins/security/public/management/role_mappings/role_mappings_grid/role_mappings_grid_page.tsx index 757e59a4e05830..cf0cb21598c62c 100644 --- a/x-pack/plugins/security/public/management/role_mappings/role_mappings_grid/role_mappings_grid_page.tsx +++ b/x-pack/plugins/security/public/management/role_mappings/role_mappings_grid/role_mappings_grid_page.tsx @@ -330,14 +330,16 @@ export class RoleMappingsGridPage extends Component { this.state.roles?.find((r) => r.name === rolename) ?? rolename; return ( - + + + ); }); - return
{roleLinks}
; + return ( + + {roleLinks} + + ); }, }, { diff --git a/x-pack/plugins/security/public/management/roles/roles_grid/roles_grid_page.test.tsx b/x-pack/plugins/security/public/management/roles/roles_grid/roles_grid_page.test.tsx index 005eebbfbf3bb3..eb6d25c9ef5ab3 100644 --- a/x-pack/plugins/security/public/management/roles/roles_grid/roles_grid_page.test.tsx +++ b/x-pack/plugins/security/public/management/roles/roles_grid/roles_grid_page.test.tsx @@ -66,6 +66,11 @@ describe('', () => { kibana: [{ base: [], spaces: [], feature: {} }], transient_metadata: { enabled: false }, }, + { + name: 'special%chars%role', + elasticsearch: { cluster: [], indices: [], run_as: [] }, + kibana: [{ base: [], spaces: [], feature: {} }], + }, ]); }); @@ -121,7 +126,7 @@ describe('', () => { expect(wrapper.find(PermissionDenied)).toMatchSnapshot(); }); - it('renders role actions as appropriate', async () => { + it('renders role actions as appropriate, escaping when necessary', async () => { const wrapper = mountWithIntl( ', () => { expect(wrapper.find(PermissionDenied)).toHaveLength(0); - const editButton = wrapper.find('EuiButtonIcon[data-test-subj="edit-role-action-test-role-1"]'); + let editButton = wrapper.find('EuiButtonIcon[data-test-subj="edit-role-action-test-role-1"]'); expect(editButton).toHaveLength(1); expect(editButton.prop('href')).toBe('/edit/test-role-1'); - const cloneButton = wrapper.find( - 'EuiButtonIcon[data-test-subj="clone-role-action-test-role-1"]' + editButton = wrapper.find( + 'EuiButtonIcon[data-test-subj="edit-role-action-special%chars%role"]' ); + expect(editButton).toHaveLength(1); + expect(editButton.prop('href')).toBe('/edit/special%25chars%25role'); + + let cloneButton = wrapper.find('EuiButtonIcon[data-test-subj="clone-role-action-test-role-1"]'); expect(cloneButton).toHaveLength(1); expect(cloneButton.prop('href')).toBe('/clone/test-role-1'); + cloneButton = wrapper.find( + 'EuiButtonIcon[data-test-subj="clone-role-action-special%chars%role"]' + ); + expect(cloneButton).toHaveLength(1); + expect(cloneButton.prop('href')).toBe('/clone/special%25chars%25role'); + expect( wrapper.find('EuiButtonIcon[data-test-subj="edit-role-action-disabled-role"]') ).toHaveLength(1); @@ -182,6 +197,11 @@ describe('', () => { kibana: [{ base: [], spaces: [], feature: {} }], metadata: { _reserved: true }, }, + { + name: 'special%chars%role', + elasticsearch: { cluster: [], indices: [], run_as: [] }, + kibana: [{ base: [], spaces: [], feature: {} }], + }, { name: 'test-role-1', elasticsearch: { cluster: [], indices: [], run_as: [] }, @@ -198,6 +218,11 @@ describe('', () => { kibana: [{ base: [], spaces: [], feature: {} }], transient_metadata: { enabled: false }, }, + { + name: 'special%chars%role', + elasticsearch: { cluster: [], indices: [], run_as: [] }, + kibana: [{ base: [], spaces: [], feature: {} }], + }, { name: 'test-role-1', elasticsearch: { cluster: [], indices: [], run_as: [] }, diff --git a/x-pack/plugins/security/public/management/roles/roles_grid/roles_grid_page.tsx b/x-pack/plugins/security/public/management/roles/roles_grid/roles_grid_page.tsx index 4c8f45d7eea439..1e733dfa0767ea 100644 --- a/x-pack/plugins/security/public/management/roles/roles_grid/roles_grid_page.tsx +++ b/x-pack/plugins/security/public/management/roles/roles_grid/roles_grid_page.tsx @@ -58,7 +58,7 @@ interface State { } const getRoleManagementHref = (action: 'edit' | 'clone', roleName?: string) => { - return `/${action}${roleName ? `/${roleName}` : ''}`; + return `/${action}${roleName ? `/${encodeURIComponent(roleName)}` : ''}`; }; export class RolesGridPage extends Component { diff --git a/x-pack/plugins/security/public/management/users/users_grid/users_grid_page.test.tsx b/x-pack/plugins/security/public/management/users/users_grid/users_grid_page.test.tsx index df8fe8cee76990..0cc3146172f708 100644 --- a/x-pack/plugins/security/public/management/users/users_grid/users_grid_page.test.tsx +++ b/x-pack/plugins/security/public/management/users/users_grid/users_grid_page.test.tsx @@ -71,6 +71,38 @@ describe('UsersGridPage', () => { expect(findTestSubject(wrapper, 'userDisabled')).toHaveLength(0); }); + it('generates valid links when usernames contain special characters', async () => { + const apiClientMock = userAPIClientMock.create(); + apiClientMock.getUsers.mockImplementation(() => { + return Promise.resolve([ + { + username: 'username with some fun characters!@#$%^&*()', + email: 'foo@bar.net', + full_name: 'foo bar', + roles: ['kibana_user'], + enabled: true, + }, + ]); + }); + + const wrapper = mountWithIntl( + + ); + + await waitForRender(wrapper); + + const link = findTestSubject(wrapper, 'userRowUserName'); + expect(link.props().href).toMatchInlineSnapshot( + `"/edit/username%20with%20some%20fun%20characters!%40%23%24%25%5E%26*()"` + ); + }); + it('renders a forbidden message if user is not authorized', async () => { const apiClient = userAPIClientMock.create(); apiClient.getUsers.mockRejectedValue({ body: { statusCode: 403 } }); diff --git a/x-pack/plugins/security/public/management/users/users_grid/users_grid_page.tsx b/x-pack/plugins/security/public/management/users/users_grid/users_grid_page.tsx index 998739a9a83af3..debe1c5e82d3ba 100644 --- a/x-pack/plugins/security/public/management/users/users_grid/users_grid_page.tsx +++ b/x-pack/plugins/security/public/management/users/users_grid/users_grid_page.tsx @@ -112,7 +112,7 @@ export class UsersGridPage extends Component { render: (username: string) => ( {username} @@ -151,14 +151,16 @@ export class UsersGridPage extends Component { const roleLinks = rolenames.map((rolename, index) => { const roleDefinition = roles?.find((role) => role.name === rolename) ?? rolename; return ( - + + + ); }); - return
{roleLinks}
; + return ( + + {roleLinks} + + ); }, }, { diff --git a/x-pack/plugins/security_solution/public/common/containers/source/index.tsx b/x-pack/plugins/security_solution/public/common/containers/source/index.tsx index c36e2de61fcbf1..2cc1c75015e074 100644 --- a/x-pack/plugins/security_solution/public/common/containers/source/index.tsx +++ b/x-pack/plugins/security_solution/public/common/containers/source/index.tsx @@ -194,15 +194,14 @@ export const useIndexFields = (sourcererScopeName: SourcererScopeName) => { const { data, notifications } = useKibana().services; const abortCtrl = useRef(new AbortController()); const dispatch = useDispatch(); - const previousIndexesName = useRef([]); - const indexNamesSelectedSelector = useMemo( () => sourcererSelectors.getIndexNamesSelectedSelector(), [] ); - const indexNames = useShallowEqualSelector((state) => - indexNamesSelectedSelector(state, sourcererScopeName) - ); + const { indexNames, previousIndexNames } = useShallowEqualSelector<{ + indexNames: string[]; + previousIndexNames: string; + }>((state) => indexNamesSelectedSelector(state, sourcererScopeName)); const setLoading = useCallback( (loading: boolean) => { @@ -230,7 +229,6 @@ export const useIndexFields = (sourcererScopeName: SourcererScopeName) => { if (!response.isPartial && !response.isRunning) { if (!didCancel) { const stringifyIndices = response.indicesExist.sort().join(); - previousIndexesName.current = response.indicesExist; dispatch( sourcererActions.setSource({ id: sourcererScopeName, @@ -279,8 +277,8 @@ export const useIndexFields = (sourcererScopeName: SourcererScopeName) => { ); useEffect(() => { - if (!isEmpty(indexNames) && !isEqual(previousIndexesName.current, indexNames)) { + if (!isEmpty(indexNames) && previousIndexNames !== indexNames.sort().join()) { indexFieldsSearch(indexNames); } - }, [indexNames, indexFieldsSearch, previousIndexesName]); + }, [indexNames, indexFieldsSearch, previousIndexNames]); }; diff --git a/x-pack/plugins/security_solution/public/common/containers/sourcerer/index.test.tsx b/x-pack/plugins/security_solution/public/common/containers/sourcerer/index.test.tsx index accfb38bc3dc1a..22cb4b91fd8393 100644 --- a/x-pack/plugins/security_solution/public/common/containers/sourcerer/index.test.tsx +++ b/x-pack/plugins/security_solution/public/common/containers/sourcerer/index.test.tsx @@ -86,7 +86,29 @@ jest.mock('../../utils/apollo_context', () => ({ })); describe('Sourcerer Hooks', () => { - const state: State = mockGlobalState; + const state: State = { + ...mockGlobalState, + sourcerer: { + ...mockGlobalState.sourcerer, + sourcererScopes: { + ...mockGlobalState.sourcerer.sourcererScopes, + [SourcererScopeName.default]: { + ...mockGlobalState.sourcerer.sourcererScopes[SourcererScopeName.default], + indexPattern: { + fields: [], + title: '', + }, + }, + [SourcererScopeName.timeline]: { + ...mockGlobalState.sourcerer.sourcererScopes[SourcererScopeName.timeline], + indexPattern: { + fields: [], + title: '', + }, + }, + }, + }, + }; const { storage } = createSecuritySolutionStorageMock(); let store = createStore( state, diff --git a/x-pack/plugins/security_solution/public/common/containers/sourcerer/index.tsx b/x-pack/plugins/security_solution/public/common/containers/sourcerer/index.tsx index b02a09625ccf3e..d9f2abeb3832e8 100644 --- a/x-pack/plugins/security_solution/public/common/containers/sourcerer/index.tsx +++ b/x-pack/plugins/security_solution/public/common/containers/sourcerer/index.tsx @@ -16,6 +16,9 @@ import { ManageScope, SourcererScopeName } from '../../store/sourcerer/model'; import { useIndexFields } from '../source'; import { State } from '../../store'; import { useUserInfo } from '../../../detections/components/user_info'; +import { timelineSelectors } from '../../../timelines/store/timeline'; +import { TimelineId } from '../../../../common/types/timeline'; +import { TimelineModel } from '../../../timelines/store/timeline/model'; export const useInitSourcerer = ( scopeId: SourcererScopeName.default | SourcererScopeName.detections = SourcererScopeName.default @@ -29,6 +32,12 @@ export const useInitSourcerer = ( ); const ConfigIndexPatterns = useSelector(getConfigIndexPatternsSelector, isEqual); + const getTimelineSelector = useMemo(() => timelineSelectors.getTimelineByIdSelector(), []); + const activeTimeline = useSelector( + (state) => getTimelineSelector(state, TimelineId.active), + isEqual + ); + useIndexFields(scopeId); useIndexFields(SourcererScopeName.timeline); @@ -40,7 +49,11 @@ export const useInitSourcerer = ( // Related to timeline useEffect(() => { - if (!loadingSignalIndex && signalIndexName != null) { + if ( + !loadingSignalIndex && + signalIndexName != null && + (activeTimeline == null || (activeTimeline != null && activeTimeline.savedObjectId == null)) + ) { dispatch( sourcererActions.setSelectedIndexPatterns({ id: SourcererScopeName.timeline, @@ -48,7 +61,7 @@ export const useInitSourcerer = ( }) ); } - }, [ConfigIndexPatterns, dispatch, loadingSignalIndex, signalIndexName]); + }, [activeTimeline, ConfigIndexPatterns, dispatch, loadingSignalIndex, signalIndexName]); // Related to the detection page useEffect(() => { diff --git a/x-pack/plugins/security_solution/public/common/store/sourcerer/actions.ts b/x-pack/plugins/security_solution/public/common/store/sourcerer/actions.ts index 0b40586798f09e..8e92d7559f1d68 100644 --- a/x-pack/plugins/security_solution/public/common/store/sourcerer/actions.ts +++ b/x-pack/plugins/security_solution/public/common/store/sourcerer/actions.ts @@ -34,3 +34,9 @@ export const setSelectedIndexPatterns = actionCreator<{ selectedPatterns: string[]; eventType?: TimelineEventsType; }>('SET_SELECTED_INDEX_PATTERNS'); + +export const initTimelineIndexPatterns = actionCreator<{ + id: SourcererScopeName; + selectedPatterns: string[]; + eventType?: TimelineEventsType; +}>('INIT_TIMELINE_INDEX_PATTERNS'); diff --git a/x-pack/plugins/security_solution/public/common/store/sourcerer/helpers.ts b/x-pack/plugins/security_solution/public/common/store/sourcerer/helpers.ts index 3ae9740cfd51dd..42a4fe73c43baa 100644 --- a/x-pack/plugins/security_solution/public/common/store/sourcerer/helpers.ts +++ b/x-pack/plugins/security_solution/public/common/store/sourcerer/helpers.ts @@ -25,16 +25,7 @@ export const createDefaultIndexPatterns = ({ eventType, id, selectedPatterns, st if (isEmpty(newSelectedPatterns)) { let defaultIndexPatterns = state.configIndexPatterns; if (id === SourcererScopeName.timeline && isEmpty(newSelectedPatterns)) { - if (eventType === 'all' && !isEmpty(state.signalIndexName)) { - defaultIndexPatterns = [...state.configIndexPatterns, state.signalIndexName ?? '']; - } else if (eventType === 'raw') { - defaultIndexPatterns = state.configIndexPatterns; - } else if ( - !isEmpty(state.signalIndexName) && - (eventType === 'signal' || eventType === 'alert') - ) { - defaultIndexPatterns = [state.signalIndexName ?? '']; - } + defaultIndexPatterns = defaultIndexPatternByEventType({ state, eventType }); } else if (id === SourcererScopeName.detections && isEmpty(newSelectedPatterns)) { defaultIndexPatterns = [state.signalIndexName ?? '']; } @@ -42,3 +33,21 @@ export const createDefaultIndexPatterns = ({ eventType, id, selectedPatterns, st } return newSelectedPatterns; }; + +export const defaultIndexPatternByEventType = ({ + state, + eventType, +}: { + state: SourcererModel; + eventType?: TimelineEventsType; +}) => { + let defaultIndexPatterns = state.configIndexPatterns; + if (eventType === 'all' && !isEmpty(state.signalIndexName)) { + defaultIndexPatterns = [...state.configIndexPatterns, state.signalIndexName ?? '']; + } else if (eventType === 'raw') { + defaultIndexPatterns = state.configIndexPatterns; + } else if (!isEmpty(state.signalIndexName) && (eventType === 'signal' || eventType === 'alert')) { + defaultIndexPatterns = [state.signalIndexName ?? '']; + } + return defaultIndexPatterns; +}; diff --git a/x-pack/plugins/security_solution/public/common/store/sourcerer/reducer.ts b/x-pack/plugins/security_solution/public/common/store/sourcerer/reducer.ts index a1112607de24fd..0c7c52e5e5733a 100644 --- a/x-pack/plugins/security_solution/public/common/store/sourcerer/reducer.ts +++ b/x-pack/plugins/security_solution/public/common/store/sourcerer/reducer.ts @@ -4,8 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -// Prefer importing entire lodash library, e.g. import { get } from "lodash" - +import { isEmpty } from 'lodash/fp'; import { reducerWithInitialState } from 'typescript-fsa-reducers'; import { @@ -14,9 +13,10 @@ import { setSelectedIndexPatterns, setSignalIndexName, setSource, + initTimelineIndexPatterns, } from './actions'; import { initialSourcererState, SourcererModel } from './model'; -import { createDefaultIndexPatterns } from './helpers'; +import { createDefaultIndexPatterns, defaultIndexPatternByEventType } from './helpers'; export type SourcererState = SourcererModel; @@ -52,6 +52,21 @@ export const sourcererReducer = reducerWithInitialState(initialSourcererState) }, }; }) + .case(initTimelineIndexPatterns, (state, { id, selectedPatterns, eventType }) => { + return { + ...state, + sourcererScopes: { + ...state.sourcererScopes, + [id]: { + ...state.sourcererScopes[id], + selectedPatterns: isEmpty(selectedPatterns) + ? defaultIndexPatternByEventType({ state, eventType }) + : selectedPatterns, + }, + }, + }; + }) + .case(setSource, (state, { id, payload }) => { const { ...sourcererScopes } = payload; return { diff --git a/x-pack/plugins/security_solution/public/common/store/sourcerer/selectors.test.ts b/x-pack/plugins/security_solution/public/common/store/sourcerer/selectors.test.ts new file mode 100644 index 00000000000000..e6577f2461a9e9 --- /dev/null +++ b/x-pack/plugins/security_solution/public/common/store/sourcerer/selectors.test.ts @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { cloneDeep } from 'lodash/fp'; +import { mockGlobalState } from '../../mock'; +import { SourcererScopeName } from './model'; +import { getSourcererScopeSelector } from './selectors'; + +describe('Sourcerer selectors', () => { + describe('getSourcererScopeSelector', () => { + it('Should exclude elastic cloud alias when selected patterns include "logs-*" as an alias', () => { + const mapStateToProps = getSourcererScopeSelector(); + expect( + mapStateToProps(mockGlobalState, SourcererScopeName.default).selectedPatterns + ).toEqual([ + 'apm-*-transaction*', + 'auditbeat-*', + 'endgame-*', + 'filebeat-*', + 'logs-*', + 'packetbeat-*', + 'winlogbeat-*', + '-*elastic-cloud-logs-*', + ]); + }); + + it('Should NOT exclude elastic cloud alias when selected patterns does NOT include "logs-*" as an alias', () => { + const mapStateToProps = getSourcererScopeSelector(); + const myMockGlobalState = cloneDeep(mockGlobalState); + myMockGlobalState.sourcerer.sourcererScopes.default.selectedPatterns = myMockGlobalState.sourcerer.sourcererScopes.default.selectedPatterns.filter( + (index) => !index.includes('logs-*') + ); + expect( + mapStateToProps(myMockGlobalState, SourcererScopeName.default).selectedPatterns + ).toEqual([ + 'apm-*-transaction*', + 'auditbeat-*', + 'endgame-*', + 'filebeat-*', + 'packetbeat-*', + 'winlogbeat-*', + ]); + }); + + it('Should NOT exclude elastic cloud alias when selected patterns include "logs-endpoint.event-*" as an alias', () => { + const mapStateToProps = getSourcererScopeSelector(); + const myMockGlobalState = cloneDeep(mockGlobalState); + myMockGlobalState.sourcerer.sourcererScopes.default.selectedPatterns = [ + ...myMockGlobalState.sourcerer.sourcererScopes.default.selectedPatterns.filter( + (index) => !index.includes('logs-*') + ), + 'logs-endpoint.event-*', + ]; + expect( + mapStateToProps(myMockGlobalState, SourcererScopeName.default).selectedPatterns + ).toEqual([ + 'apm-*-transaction*', + 'auditbeat-*', + 'endgame-*', + 'filebeat-*', + 'packetbeat-*', + 'winlogbeat-*', + 'logs-endpoint.event-*', + ]); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/public/common/store/sourcerer/selectors.ts b/x-pack/plugins/security_solution/public/common/store/sourcerer/selectors.ts index ca9ea26ba5bac0..e7bd6234cb207b 100644 --- a/x-pack/plugins/security_solution/public/common/store/sourcerer/selectors.ts +++ b/x-pack/plugins/security_solution/public/common/store/sourcerer/selectors.ts @@ -41,13 +41,18 @@ export const getIndexNamesSelectedSelector = () => { const getScopesSelector = scopesSelector(); const getConfigIndexPatternsSelector = configIndexPatternsSelector(); - const mapStateToProps = (state: State, scopeId: SourcererScopeName): string[] => { + const mapStateToProps = ( + state: State, + scopeId: SourcererScopeName + ): { indexNames: string[]; previousIndexNames: string } => { const scope = getScopesSelector(state)[scopeId]; const configIndexPatterns = getConfigIndexPatternsSelector(state); - - return scope.selectedPatterns.length === 0 ? configIndexPatterns : scope.selectedPatterns; + return { + indexNames: + scope.selectedPatterns.length === 0 ? configIndexPatterns : scope.selectedPatterns, + previousIndexNames: scope.indexPattern.title, + }; }; - return mapStateToProps; }; @@ -81,11 +86,18 @@ export const defaultIndexNamesSelector = () => { return mapStateToProps; }; +const EXLCUDE_ELASTIC_CLOUD_INDEX = '-*elastic-cloud-logs-*'; export const getSourcererScopeSelector = () => { const getScopesSelector = scopesSelector(); - const mapStateToProps = (state: State, scopeId: SourcererScopeName): ManageScope => - getScopesSelector(state)[scopeId]; + const mapStateToProps = (state: State, scopeId: SourcererScopeName): ManageScope => ({ + ...getScopesSelector(state)[scopeId], + selectedPatterns: getScopesSelector(state)[scopeId].selectedPatterns.some( + (index) => index === 'logs-*' + ) + ? [...getScopesSelector(state)[scopeId].selectedPatterns, EXLCUDE_ELASTIC_CLOUD_INDEX] + : getScopesSelector(state)[scopeId].selectedPatterns, + }); return mapStateToProps; }; diff --git a/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.test.tsx b/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.test.tsx index bfc104b1052366..ecc0fc54d0d47a 100644 --- a/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.test.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.test.tsx @@ -47,7 +47,9 @@ describe('alert actions', () => { searchStrategyClient = { aggs: {} as ISearchStart['aggs'], showError: jest.fn(), - search: jest.fn().mockResolvedValue({ data: mockTimelineDetails }), + search: jest + .fn() + .mockImplementation(() => ({ toPromise: () => ({ data: mockTimelineDetails }) })), searchSource: {} as ISearchStart['searchSource'], session: dataPluginMock.createStartContract().search.session, }; @@ -400,6 +402,78 @@ describe('alert actions', () => { expect(createTimeline).toHaveBeenCalledWith(defaultTimelineProps); }); }); + + describe('Eql', () => { + test(' with signal.group.id', async () => { + const ecsDataMock: Ecs = { + ...mockEcsDataWithAlert, + signal: { + rule: { + ...mockEcsDataWithAlert.signal?.rule!, + type: ['eql'], + timeline_id: [''], + }, + group: { + id: ['my-group-id'], + }, + }, + }; + + await sendAlertToTimelineAction({ + createTimeline, + ecsData: ecsDataMock, + nonEcsData: [], + updateTimelineIsLoading, + searchStrategyClient, + }); + + expect(updateTimelineIsLoading).not.toHaveBeenCalled(); + expect(createTimeline).toHaveBeenCalledTimes(1); + expect(createTimeline).toHaveBeenCalledWith({ + ...defaultTimelineProps, + timeline: { + ...defaultTimelineProps.timeline, + dataProviders: [ + { + and: [], + enabled: true, + excluded: false, + id: + 'send-alert-to-timeline-action-default-draggable-event-details-value-formatted-field-value-timeline-1-alert-id-my-group-id', + kqlQuery: '', + name: '1', + queryMatch: { field: 'signal.group.id', operator: ':', value: 'my-group-id' }, + }, + ], + }, + }); + }); + + test(' with NO signal.group.id', async () => { + const ecsDataMock: Ecs = { + ...mockEcsDataWithAlert, + signal: { + rule: { + ...mockEcsDataWithAlert.signal?.rule!, + type: ['eql'], + timeline_id: [''], + }, + }, + }; + + await sendAlertToTimelineAction({ + createTimeline, + ecsData: ecsDataMock, + nonEcsData: [], + updateTimelineIsLoading, + searchStrategyClient, + }); + + expect(updateTimelineIsLoading).not.toHaveBeenCalled(); + expect(createTimeline).toHaveBeenCalledTimes(1); + expect(createTimeline).toHaveBeenCalledWith(defaultTimelineProps); + }); + }); }); describe('determineToAndFrom', () => { diff --git a/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.tsx b/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.tsx index 043a5afc4480d7..e3defaea2ec673 100644 --- a/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/alerts_table/actions.tsx @@ -150,8 +150,10 @@ export const getThresholdAggregationDataProvider = ( ]; }; -export const isEqlRule = (ecsData: Ecs) => - ecsData.signal?.rule?.type?.length && ecsData.signal?.rule?.type[0] === 'eql'; +export const isEqlRuleWithGroupId = (ecsData: Ecs) => + ecsData.signal?.rule?.type?.length && + ecsData.signal?.rule?.type[0] === 'eql' && + ecsData.signal?.group?.id?.length; export const isThresholdRule = (ecsData: Ecs) => ecsData.signal?.rule?.type?.length && ecsData.signal?.rule?.type[0] === 'threshold'; @@ -181,24 +183,23 @@ export const sendAlertToTimelineAction = async ({ timelineType: TimelineType.template, }, }), - searchStrategyClient.search< - TimelineEventsDetailsRequestOptions, - TimelineEventsDetailsStrategyResponse - >( - { - defaultIndex: [], - docValueFields: [], - indexName: ecsData._index ?? '', - eventId: ecsData._id, - factoryQueryType: TimelineEventsQueries.details, - }, - { - strategy: 'securitySolutionTimelineSearchStrategy', - } - ), + searchStrategyClient + .search( + { + defaultIndex: [], + docValueFields: [], + indexName: ecsData._index ?? '', + eventId: ecsData._id, + factoryQueryType: TimelineEventsQueries.details, + }, + { + strategy: 'securitySolutionTimelineSearchStrategy', + } + ) + .toPromise(), ]); const resultingTimeline: TimelineResult = getOr({}, 'data.getOneTimeline', responseTimeline); - const eventData: TimelineEventsDetailsItem[] = getOr([], 'data', eventDataResp); + const eventData: TimelineEventsDetailsItem[] = eventDataResp.data ?? []; if (!isEmpty(resultingTimeline)) { const timelineTemplate: TimelineResult = omitTypenameInTimeline(resultingTimeline); const { timeline, notes } = formatTimelineResultToModel( @@ -327,7 +328,7 @@ export const sendAlertToTimelineAction = async ({ }, }, ]; - if (isEqlRule(ecsData)) { + if (isEqlRuleWithGroupId(ecsData)) { const signalGroupId = ecsData.signal?.group?.id?.length ? ecsData.signal?.group?.id[0] : 'unknown-signal-group-id'; diff --git a/x-pack/plugins/security_solution/public/detections/components/user_info/index.test.tsx b/x-pack/plugins/security_solution/public/detections/components/user_info/index.test.tsx index 9b15007136b2e5..e87303efbe526b 100644 --- a/x-pack/plugins/security_solution/public/detections/components/user_info/index.test.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/user_info/index.test.tsx @@ -42,7 +42,7 @@ describe('useUserInfo', () => { isSignalIndexExists: null, loading: true, signalIndexName: null, - signalIndexTemplateOutdated: null, + signalIndexMappingOutdated: null, }, error: undefined, }); @@ -53,7 +53,7 @@ describe('useUserInfo', () => { const spyOnCreateSignalIndex = jest.spyOn(api, 'createSignalIndex'); const spyOnGetSignalIndex = jest.spyOn(api, 'getSignalIndex').mockResolvedValueOnce({ name: 'mock-signal-index', - template_outdated: true, + index_mapping_outdated: true, }); await act(async () => { const { waitForNextUpdate } = renderHook(() => useUserInfo(), { wrapper: ManageUserInfo }); diff --git a/x-pack/plugins/security_solution/public/detections/components/user_info/index.tsx b/x-pack/plugins/security_solution/public/detections/components/user_info/index.tsx index ac2bf438d7fa69..3b0976f459324c 100644 --- a/x-pack/plugins/security_solution/public/detections/components/user_info/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/user_info/index.tsx @@ -20,7 +20,7 @@ export interface State { hasEncryptionKey: boolean | null; loading: boolean; signalIndexName: string | null; - signalIndexTemplateOutdated: boolean | null; + signalIndexMappingOutdated: boolean | null; } export const initialState: State = { @@ -32,7 +32,7 @@ export const initialState: State = { hasEncryptionKey: null, loading: true, signalIndexName: null, - signalIndexTemplateOutdated: null, + signalIndexMappingOutdated: null, }; export type Action = @@ -66,8 +66,8 @@ export type Action = signalIndexName: string | null; } | { - type: 'updateSignalIndexTemplateOutdated'; - signalIndexTemplateOutdated: boolean | null; + type: 'updateSignalIndexMappingOutdated'; + signalIndexMappingOutdated: boolean | null; }; export const userInfoReducer = (state: State, action: Action): State => { @@ -120,10 +120,10 @@ export const userInfoReducer = (state: State, action: Action): State => { signalIndexName: action.signalIndexName, }; } - case 'updateSignalIndexTemplateOutdated': { + case 'updateSignalIndexMappingOutdated': { return { ...state, - signalIndexTemplateOutdated: action.signalIndexTemplateOutdated, + signalIndexMappingOutdated: action.signalIndexMappingOutdated, }; } default: @@ -156,7 +156,7 @@ export const useUserInfo = (): State => { hasEncryptionKey, loading, signalIndexName, - signalIndexTemplateOutdated, + signalIndexMappingOutdated, }, dispatch, ] = useUserData(); @@ -171,7 +171,7 @@ export const useUserInfo = (): State => { loading: indexNameLoading, signalIndexExists: isApiSignalIndexExists, signalIndexName: apiSignalIndexName, - signalIndexTemplateOutdated: apiSignalIndexTemplateOutdated, + signalIndexMappingOutdated: apiSignalIndexMappingOutdated, createDeSignalIndex: createSignalIndex, } = useSignalIndex(); @@ -234,15 +234,15 @@ export const useUserInfo = (): State => { useEffect(() => { if ( !loading && - signalIndexTemplateOutdated !== apiSignalIndexTemplateOutdated && - apiSignalIndexTemplateOutdated != null + signalIndexMappingOutdated !== apiSignalIndexMappingOutdated && + apiSignalIndexMappingOutdated != null ) { dispatch({ - type: 'updateSignalIndexTemplateOutdated', - signalIndexTemplateOutdated: apiSignalIndexTemplateOutdated, + type: 'updateSignalIndexMappingOutdated', + signalIndexMappingOutdated: apiSignalIndexMappingOutdated, }); } - }, [dispatch, loading, signalIndexTemplateOutdated, apiSignalIndexTemplateOutdated]); + }, [dispatch, loading, signalIndexMappingOutdated, apiSignalIndexMappingOutdated]); useEffect(() => { if ( @@ -250,7 +250,7 @@ export const useUserInfo = (): State => { hasEncryptionKey && hasIndexManage && ((isSignalIndexExists != null && !isSignalIndexExists) || - (signalIndexTemplateOutdated != null && signalIndexTemplateOutdated)) && + (signalIndexMappingOutdated != null && signalIndexMappingOutdated)) && createSignalIndex != null ) { createSignalIndex(); @@ -261,7 +261,7 @@ export const useUserInfo = (): State => { hasEncryptionKey, isSignalIndexExists, hasIndexManage, - signalIndexTemplateOutdated, + signalIndexMappingOutdated, ]); return { @@ -273,6 +273,6 @@ export const useUserInfo = (): State => { hasIndexManage, hasIndexWrite, signalIndexName, - signalIndexTemplateOutdated, + signalIndexMappingOutdated, }; }; diff --git a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/mock.ts b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/mock.ts index 4fd240348f0f3b..21b561ec9cddb9 100644 --- a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/mock.ts +++ b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/mock.ts @@ -980,7 +980,7 @@ export const mockStatusAlertQuery: object = { export const mockSignalIndex: AlertsIndex = { name: 'mock-signal-index', - template_outdated: false, + index_mapping_outdated: false, }; export const mockUserPrivilege: Privilege = { diff --git a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/types.ts b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/types.ts index 59ab416ecc824b..dadeb1e7958b56 100644 --- a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/types.ts +++ b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/types.ts @@ -44,7 +44,7 @@ export interface UpdateAlertStatusProps { export interface AlertsIndex { name: string; - template_outdated: boolean; + index_mapping_outdated: boolean; } export interface Privilege { diff --git a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/use_signal_index.test.tsx b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/use_signal_index.test.tsx index 1db952526414ac..07375a31f3bbc9 100644 --- a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/use_signal_index.test.tsx +++ b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/use_signal_index.test.tsx @@ -26,7 +26,7 @@ describe('useSignalIndex', () => { loading: true, signalIndexExists: null, signalIndexName: null, - signalIndexTemplateOutdated: null, + signalIndexMappingOutdated: null, }); }); }); @@ -43,7 +43,7 @@ describe('useSignalIndex', () => { loading: false, signalIndexExists: true, signalIndexName: 'mock-signal-index', - signalIndexTemplateOutdated: false, + signalIndexMappingOutdated: false, }); }); }); @@ -64,7 +64,7 @@ describe('useSignalIndex', () => { loading: false, signalIndexExists: true, signalIndexName: 'mock-signal-index', - signalIndexTemplateOutdated: false, + signalIndexMappingOutdated: false, }); }); }); @@ -104,7 +104,7 @@ describe('useSignalIndex', () => { loading: false, signalIndexExists: false, signalIndexName: null, - signalIndexTemplateOutdated: null, + signalIndexMappingOutdated: null, }); }); }); @@ -125,7 +125,7 @@ describe('useSignalIndex', () => { loading: false, signalIndexExists: false, signalIndexName: null, - signalIndexTemplateOutdated: null, + signalIndexMappingOutdated: null, }); }); }); diff --git a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/use_signal_index.tsx b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/use_signal_index.tsx index f7d2202736169d..1233456359b7fb 100644 --- a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/use_signal_index.tsx +++ b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/use_signal_index.tsx @@ -17,7 +17,7 @@ export interface ReturnSignalIndex { loading: boolean; signalIndexExists: boolean | null; signalIndexName: string | null; - signalIndexTemplateOutdated: boolean | null; + signalIndexMappingOutdated: boolean | null; createDeSignalIndex: Func | null; } @@ -31,7 +31,7 @@ export const useSignalIndex = (): ReturnSignalIndex => { const [signalIndex, setSignalIndex] = useState>({ signalIndexExists: null, signalIndexName: null, - signalIndexTemplateOutdated: null, + signalIndexMappingOutdated: null, createDeSignalIndex: null, }); const [, dispatchToaster] = useStateToaster(); @@ -49,7 +49,7 @@ export const useSignalIndex = (): ReturnSignalIndex => { setSignalIndex({ signalIndexExists: true, signalIndexName: signal.name, - signalIndexTemplateOutdated: signal.template_outdated, + signalIndexMappingOutdated: signal.index_mapping_outdated, createDeSignalIndex: createIndex, }); } @@ -58,7 +58,7 @@ export const useSignalIndex = (): ReturnSignalIndex => { setSignalIndex({ signalIndexExists: false, signalIndexName: null, - signalIndexTemplateOutdated: null, + signalIndexMappingOutdated: null, createDeSignalIndex: createIndex, }); if (isSecurityAppError(error) && error.body.status_code !== 404) { @@ -89,7 +89,7 @@ export const useSignalIndex = (): ReturnSignalIndex => { setSignalIndex({ signalIndexExists: false, signalIndexName: null, - signalIndexTemplateOutdated: null, + signalIndexMappingOutdated: null, createDeSignalIndex: createIndex, }); errorToToaster({ title: i18n.SIGNAL_POST_FAILURE, error, dispatchToaster }); diff --git a/x-pack/plugins/security_solution/public/management/components/management_empty_state.tsx b/x-pack/plugins/security_solution/public/management/components/management_empty_state.tsx index d51a23639f5cbb..18745897c594f4 100644 --- a/x-pack/plugins/security_solution/public/management/components/management_empty_state.tsx +++ b/x-pack/plugins/security_solution/public/management/components/management_empty_state.tsx @@ -66,7 +66,7 @@ const PolicyEmptyState = React.memo<{ diff --git a/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_forms/protections/malware.tsx b/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_forms/protections/malware.tsx index 1da832fb081efc..6773ed65419275 100644 --- a/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_forms/protections/malware.tsx +++ b/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_forms/protections/malware.tsx @@ -11,6 +11,7 @@ import { EuiRadio, EuiSwitch, EuiTitle, + EuiText, EuiSpacer, htmlIdGenerator, EuiCallOut, @@ -28,6 +29,7 @@ import { policyConfig } from '../../../store/policy_details/selectors'; import { usePolicyDetailsSelector } from '../../policy_hooks'; import { clone } from '../../../models/policy_details_config'; import { LinkToApp } from '../../../../../../common/components/endpoint/link_to_app'; +import { popupVersionsMap } from './popup_options_to_versions'; const ProtectionRadioGroup = styled.div` display: flex; @@ -83,6 +85,25 @@ const ProtectionRadio = React.memo(({ id, label }: { id: ProtectionModes; label: ProtectionRadio.displayName = 'ProtectionRadio'; +const SupportedVersionNotice = ({ optionName }: { optionName: string }) => { + const version = popupVersionsMap.get(optionName); + if (!version) { + return null; + } + + return ( + + + + + + ); +}; + /** The Malware Protections form for policy details * which will configure for all relevant OSes. */ @@ -189,14 +210,15 @@ export const MalwareProtections = React.memo(() => { /> + + ); diff --git a/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_forms/protections/popup_options_to_versions.ts b/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_forms/protections/popup_options_to_versions.ts new file mode 100644 index 00000000000000..d4c7d0102ebd46 --- /dev/null +++ b/x-pack/plugins/security_solution/public/management/pages/policy/view/policy_forms/protections/popup_options_to_versions.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +const popupVersions: Array<[string, string]> = [['malware', '7.11+']]; + +export const popupVersionsMap: ReadonlyMap = new Map(popupVersions); diff --git a/x-pack/plugins/security_solution/public/overview/pages/overview.tsx b/x-pack/plugins/security_solution/public/overview/pages/overview.tsx index 5a3b4ec3846869..a292ec3e1a1194 100644 --- a/x-pack/plugins/security_solution/public/overview/pages/overview.tsx +++ b/x-pack/plugins/security_solution/public/overview/pages/overview.tsx @@ -131,7 +131,7 @@ const OverviewComponent: React.FC = ({ ( - { indices: defaultIndicesName, onlyCheckIfIndicesExist: false }, + { indices: defaultIndicesName, onlyCheckIfIndicesExist: true }, { strategy: 'securitySolutionIndexFields', } diff --git a/x-pack/plugins/security_solution/public/timelines/components/edit_data_provider/index.tsx b/x-pack/plugins/security_solution/public/timelines/components/edit_data_provider/index.tsx index 2bc202c65f6ab4..8127f2be13d4e6 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/edit_data_provider/index.tsx +++ b/x-pack/plugins/security_solution/public/timelines/components/edit_data_provider/index.tsx @@ -189,7 +189,7 @@ export const StatefulEditDataProvider = React.memo( - + ( - + {type !== DataProviderType.template && @@ -245,7 +245,7 @@ export const StatefulEditDataProvider = React.memo( ) : null} - + @@ -265,7 +265,7 @@ export const StatefulEditDataProvider = React.memo( }) || isValueFieldInvalid } onClick={handleSave} - size="s" + size="m" > {i18n.SAVE} diff --git a/x-pack/plugins/security_solution/public/timelines/components/open_timeline/helpers.ts b/x-pack/plugins/security_solution/public/timelines/components/open_timeline/helpers.ts index c89740f667b29b..4c3be81a4992ad 100644 --- a/x-pack/plugins/security_solution/public/timelines/components/open_timeline/helpers.ts +++ b/x-pack/plugins/security_solution/public/timelines/components/open_timeline/helpers.ts @@ -378,7 +378,7 @@ export const dispatchUpdateTimeline = (dispatch: Dispatch): DispatchUpdateTimeli ruleNote, }: UpdateTimeline): (() => void) => () => { dispatch( - sourcererActions.setSelectedIndexPatterns({ + sourcererActions.initTimelineIndexPatterns({ id: SourcererScopeName.timeline, selectedPatterns: timeline.indexNames, eventType: timeline.eventType, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/check_template_version.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/check_template_version.ts index 473a2dad37f196..e7618f155967bc 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/check_template_version.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/check_template_version.ts @@ -6,20 +6,19 @@ import { get } from 'lodash'; import { LegacyAPICaller } from '../../../../../../../../src/core/server'; -import { getSignalsTemplate } from './get_signals_template'; import { getTemplateExists } from '../../index/get_template_exists'; +import { SIGNALS_TEMPLATE_VERSION } from './get_signals_template'; export const templateNeedsUpdate = async (callCluster: LegacyAPICaller, index: string) => { const templateExists = await getTemplateExists(callCluster, index); - let existingTemplateVersion: number | undefined; - if (templateExists) { - const existingTemplate: unknown = await callCluster('indices.getTemplate', { - name: index, - }); - existingTemplateVersion = get(existingTemplate, [index, 'version']); + if (!templateExists) { + return true; } - const newTemplate = getSignalsTemplate(index); - if (existingTemplateVersion === undefined || existingTemplateVersion < newTemplate.version) { + const existingTemplate: unknown = await callCluster('indices.getTemplate', { + name: index, + }); + const existingTemplateVersion: number | undefined = get(existingTemplate, [index, 'version']); + if (existingTemplateVersion === undefined || existingTemplateVersion < SIGNALS_TEMPLATE_VERSION) { return true; } return false; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/create_index_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/create_index_route.ts index 070a396d747046..287459cf5ec9ac 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/create_index_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/create_index_route.ts @@ -12,10 +12,11 @@ import { getIndexExists } from '../../index/get_index_exists'; import { getPolicyExists } from '../../index/get_policy_exists'; import { setPolicy } from '../../index/set_policy'; import { setTemplate } from '../../index/set_template'; -import { getSignalsTemplate } from './get_signals_template'; +import { getSignalsTemplate, SIGNALS_TEMPLATE_VERSION } from './get_signals_template'; import { createBootstrapIndex } from '../../index/create_bootstrap_index'; import signalsPolicy from './signals_policy.json'; import { templateNeedsUpdate } from './check_template_version'; +import { getIndexVersion } from './get_index_version'; export const createIndexRoute = (router: IRouter) => { router.post( @@ -67,18 +68,20 @@ export const createDetectionIndex = async ( } const index = siemClient.getSignalsIndex(); - const indexExists = await getIndexExists(callCluster, index); + const policyExists = await getPolicyExists(callCluster, index); + if (!policyExists) { + await setPolicy(callCluster, index, signalsPolicy); + } if (await templateNeedsUpdate(callCluster, index)) { - const policyExists = await getPolicyExists(callCluster, index); - if (!policyExists) { - await setPolicy(callCluster, index, signalsPolicy); - } await setTemplate(callCluster, index, getSignalsTemplate(index)); - if (indexExists) { + } + const indexExists = await getIndexExists(callCluster, index); + if (indexExists) { + const indexVersion = await getIndexVersion(callCluster, index); + if (indexVersion !== SIGNALS_TEMPLATE_VERSION) { await callCluster('indices.rollover', { alias: index }); } - } - if (!indexExists) { + } else { await createBootstrapIndex(callCluster, index); } }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_index_version.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_index_version.ts new file mode 100644 index 00000000000000..062cffd3935554 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_index_version.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { get } from 'lodash'; +import { LegacyAPICaller } from '../../../../../../../../src/core/server'; +import { readIndex } from '../../index/read_index'; + +interface IndicesAliasResponse { + [index: string]: IndexAliasResponse; +} + +interface IndexAliasResponse { + aliases: { + [aliasName: string]: Record; + }; +} + +export const getIndexVersion = async ( + callCluster: LegacyAPICaller, + index: string +): Promise => { + const indexAlias: IndicesAliasResponse = await callCluster('indices.getAlias', { + index, + }); + const writeIndex = Object.keys(indexAlias).find( + (key) => indexAlias[key].aliases[index].is_write_index + ); + if (writeIndex === undefined) { + return undefined; + } + const writeIndexMapping = await readIndex(callCluster, writeIndex); + return get(writeIndexMapping, [writeIndex, 'mappings', '_meta', 'version']); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.ts index b676ab5705bfc1..d1a9b701b2c9d8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.ts @@ -7,8 +7,10 @@ import signalsMapping from './signals_mapping.json'; import ecsMapping from './ecs_mapping.json'; +export const SIGNALS_TEMPLATE_VERSION = 2; +export const MIN_EQL_RULE_INDEX_VERSION = 2; + export const getSignalsTemplate = (index: string) => { - const version = 2; const template = { settings: { index: { @@ -31,10 +33,10 @@ export const getSignalsTemplate = (index: string) => { signal: signalsMapping.mappings.properties.signal, }, _meta: { - version, + version: SIGNALS_TEMPLATE_VERSION, }, }, - version, + version: SIGNALS_TEMPLATE_VERSION, }; return template; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/read_index_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/read_index_route.ts index b9ae8b546b8bd9..d1b1a2b4dd0eb0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/read_index_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/read_index_route.ts @@ -8,7 +8,8 @@ import { IRouter } from '../../../../../../../../src/core/server'; import { DETECTION_ENGINE_INDEX_URL } from '../../../../../common/constants'; import { transformError, buildSiemResponse } from '../utils'; import { getIndexExists } from '../../index/get_index_exists'; -import { templateNeedsUpdate } from './check_template_version'; +import { SIGNALS_TEMPLATE_VERSION } from './get_signals_template'; +import { getIndexVersion } from './get_index_version'; export const readIndexRoute = (router: IRouter) => { router.get( @@ -32,10 +33,24 @@ export const readIndexRoute = (router: IRouter) => { const index = siemClient.getSignalsIndex(); const indexExists = await getIndexExists(clusterClient.callAsCurrentUser, index); - const templateOutdated = await templateNeedsUpdate(clusterClient.callAsCurrentUser, index); if (indexExists) { - return response.ok({ body: { name: index, template_outdated: templateOutdated } }); + let mappingOutdated: boolean | null = null; + try { + const indexVersion = await getIndexVersion(clusterClient.callAsCurrentUser, index); + mappingOutdated = indexVersion !== SIGNALS_TEMPLATE_VERSION; + } catch (err) { + const error = transformError(err); + // Some users may not have the view_index_metadata permission necessary to check the index mapping version + // so just continue and return null for index_mapping_outdated if the error is a 403 + if (error.statusCode !== 403) { + return siemResponse.error({ + body: error.message, + statusCode: error.statusCode, + }); + } + } + return response.ok({ body: { name: index, index_mapping_outdated: mappingOutdated } }); } else { return siemResponse.error({ statusCode: 404, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/utils.test.ts index 6768e9534a87ed..977dad680f8a40 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/utils.test.ts @@ -7,7 +7,9 @@ import Boom from 'boom'; import { SavedObjectsFindResponse } from 'kibana/server'; -import { IRuleSavedAttributesSavedObjectAttributes, IRuleStatusAttributes } from '../rules/types'; + +import { alertsClientMock } from '../../../../../alerts/server/mocks'; +import { IRuleSavedAttributesSavedObjectAttributes, IRuleStatusSOAttributes } from '../rules/types'; import { BadRequestError } from '../errors/bad_request_error'; import { transformError, @@ -19,8 +21,14 @@ import { transformImportError, convertToSnakeCase, SiemResponseFactory, + mergeStatuses, + getFailingRules, } from './utils'; import { responseMock } from './__mocks__'; +import { exampleRuleStatus, exampleFindRuleStatusResponse } from '../signals/__mocks__/es_results'; +import { getResult } from './__mocks__/request_responses'; + +let alertsClient: ReturnType; describe('utils', () => { describe('transformError', () => { @@ -319,7 +327,7 @@ describe('utils', () => { saved_objects: [], }; expect( - convertToSnakeCase(values.saved_objects[0]?.attributes) // this is undefined, but it says it's not + convertToSnakeCase(values.saved_objects[0]?.attributes) // this is undefined, but it says it's not ).toEqual(null); }); }); @@ -350,4 +358,133 @@ describe('utils', () => { ); }); }); + + describe('mergeStatuses', () => { + it('merges statuses and converts from camelCase saved object to snake_case HTTP response', () => { + const statusOne = exampleRuleStatus(); + statusOne.attributes.status = 'failed'; + const statusTwo = exampleRuleStatus(); + statusTwo.attributes.status = 'failed'; + const currentStatus = exampleRuleStatus(); + const foundRules = exampleFindRuleStatusResponse([currentStatus, statusOne, statusTwo]); + const res = mergeStatuses(currentStatus.attributes.alertId, foundRules.saved_objects, { + 'myfakealertid-8cfac': { + current_status: { + alert_id: 'myfakealertid-8cfac', + status_date: '2020-03-27T22:55:59.517Z', + status: 'succeeded', + last_failure_at: null, + last_success_at: '2020-03-27T22:55:59.517Z', + last_failure_message: null, + last_success_message: 'succeeded', + gap: null, + bulk_create_time_durations: [], + search_after_time_durations: [], + last_look_back_date: null, + }, + failures: [], + }, + }); + expect(res).toEqual({ + 'myfakealertid-8cfac': { + current_status: { + alert_id: 'myfakealertid-8cfac', + status_date: '2020-03-27T22:55:59.517Z', + status: 'succeeded', + last_failure_at: null, + last_success_at: '2020-03-27T22:55:59.517Z', + last_failure_message: null, + last_success_message: 'succeeded', + gap: null, + bulk_create_time_durations: [], + search_after_time_durations: [], + last_look_back_date: null, + }, + failures: [], + }, + 'f4b8e31d-cf93-4bde-a265-298bde885cd7': { + current_status: { + alert_id: 'f4b8e31d-cf93-4bde-a265-298bde885cd7', + status_date: '2020-03-27T22:55:59.517Z', + status: 'succeeded', + last_failure_at: null, + last_success_at: '2020-03-27T22:55:59.517Z', + last_failure_message: null, + last_success_message: 'succeeded', + gap: null, + bulk_create_time_durations: [], + search_after_time_durations: [], + last_look_back_date: null, + }, + failures: [ + { + alert_id: 'f4b8e31d-cf93-4bde-a265-298bde885cd7', + status_date: '2020-03-27T22:55:59.517Z', + status: 'failed', + last_failure_at: null, + last_success_at: '2020-03-27T22:55:59.517Z', + last_failure_message: null, + last_success_message: 'succeeded', + gap: null, + bulk_create_time_durations: [], + search_after_time_durations: [], + last_look_back_date: null, + }, + { + alert_id: 'f4b8e31d-cf93-4bde-a265-298bde885cd7', + status_date: '2020-03-27T22:55:59.517Z', + status: 'failed', + last_failure_at: null, + last_success_at: '2020-03-27T22:55:59.517Z', + last_failure_message: null, + last_success_message: 'succeeded', + gap: null, + bulk_create_time_durations: [], + search_after_time_durations: [], + last_look_back_date: null, + }, + ], + }, + }); + }); + }); + + describe('getFailingRules', () => { + beforeEach(() => { + alertsClient = alertsClientMock.create(); + }); + it('getFailingRules finds no failing rules', async () => { + alertsClient.get.mockResolvedValue(getResult()); + const res = await getFailingRules(['my-fake-id'], alertsClient); + expect(res).toEqual({}); + }); + it('getFailingRules finds a failing rule', async () => { + const foundRule = getResult(); + foundRule.executionStatus = { + status: 'error', + lastExecutionDate: foundRule.executionStatus.lastExecutionDate, + error: { + reason: 'read', + message: 'oops', + }, + }; + alertsClient.get.mockResolvedValue(foundRule); + const res = await getFailingRules([foundRule.id], alertsClient); + expect(res).toEqual({ [foundRule.id]: foundRule }); + }); + it('getFailingRules throws an error', async () => { + alertsClient.get.mockImplementation(() => { + throw new Error('my test error'); + }); + let error; + try { + await getFailingRules(['my-fake-id'], alertsClient); + } catch (exc) { + error = exc; + } + expect(error.message).toEqual( + 'Failed to get executionStatus with AlertsClient: my test error' + ); + }); + }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/utils.ts index 96f96d7ebcc9ee..72be7a3c0fa08b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/utils.ts @@ -17,7 +17,7 @@ import { } from '../../../../../../../src/core/server'; import { AlertsClient } from '../../../../../alerts/server'; import { BadRequestError } from '../errors/bad_request_error'; -import { RuleStatusResponse, IRuleStatusAttributes } from '../rules/types'; +import { RuleStatusResponse, IRuleStatusSOAttributes } from '../rules/types'; export interface OutputError { message: string; @@ -294,39 +294,53 @@ export const convertToSnakeCase = >( }, {}); }; +/** + * + * @param id rule id + * @param currentStatusAndFailures array of rule statuses where the 0th status is the current status and 1-5 positions are the historical failures + * @param acc accumulated rule id : statuses + */ export const mergeStatuses = ( id: string, - failures: Array>, + currentStatusAndFailures: Array>, acc: RuleStatusResponse -) => { - if (failures.length === 0) { +): RuleStatusResponse => { + if (currentStatusAndFailures.length === 0) { return { ...acc, }; } - const convertedCurrentStatus = convertToSnakeCase(failures[0].attributes); + const convertedCurrentStatus = convertToSnakeCase( + currentStatusAndFailures[0].attributes + ); return { ...acc, [id]: { current_status: convertedCurrentStatus, - failures: failures.map((errorItem) => - convertToSnakeCase(errorItem.attributes) - ), + failures: currentStatusAndFailures + .slice(1) + .map((errorItem) => convertToSnakeCase(errorItem.attributes)), }, } as RuleStatusResponse; }; -export const getFailingRules = (ids: string[], alertsClient: AlertsClient) => - Promise.all( - ids.map(async (id) => - alertsClient.get({ - id, - }) - ) - ) - .then((rules) => rules.filter((rule) => rule.executionStatus.status === 'error')) - .then((rules) => - rules.reduce((acc, failingRule) => { +export type GetFailingRulesResult = Record; + +export const getFailingRules = async ( + ids: string[], + alertsClient: AlertsClient +): Promise => { + try { + const errorRules = await Promise.all( + ids.map(async (id) => + alertsClient.get({ + id, + }) + ) + ); + return errorRules + .filter((rule) => rule.executionStatus.status === 'error') + .reduce((acc, failingRule) => { const accum = acc; const theRule = failingRule; return { @@ -335,5 +349,8 @@ export const getFailingRules = (ids: string[], alertsClient: AlertsClient) => }, ...accum, }; - }, {} as Record) - ); + }, {}); + } catch (exc) { + throw new Error(`Failed to get executionStatus with AlertsClient: ${exc.message}`); + } +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/apm_403_response_to_a_post.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/apm_403_response_to_a_post.json index 9139ca82cc7d8b..44deb001406549 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/apm_403_response_to_a_post.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/apm_403_response_to_a_post.json @@ -20,9 +20,9 @@ "rule_id": "a87a4e42-1d82-44bd-b0bf-d9b7f91fb89e", "severity": "medium", "tags": [ - "APM", - "Elastic" + "Elastic", + "APM" ], "type": "query", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/apm_405_response_method_not_allowed.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/apm_405_response_method_not_allowed.json index 2eb7d711e5fb83..5ff5cb39a6b211 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/apm_405_response_method_not_allowed.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/apm_405_response_method_not_allowed.json @@ -20,9 +20,9 @@ "rule_id": "75ee75d8-c180-481c-ba88-ee50129a6aef", "severity": "medium", "tags": [ - "APM", - "Elastic" + "Elastic", + "APM" ], "type": "query", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/apm_null_user_agent.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/apm_null_user_agent.json index e78395be8fb1b0..75bbfe477732b0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/apm_null_user_agent.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/apm_null_user_agent.json @@ -38,9 +38,9 @@ "rule_id": "43303fd4-4839-4e48-b2b2-803ab060758d", "severity": "medium", "tags": [ - "APM", - "Elastic" + "Elastic", + "APM" ], "type": "query", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/apm_sqlmap_user_agent.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/apm_sqlmap_user_agent.json index aaaab6b5c6031f..10185c28a8e52f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/apm_sqlmap_user_agent.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/apm_sqlmap_user_agent.json @@ -20,9 +20,9 @@ "rule_id": "d49cc73f-7a16-4def-89ce-9fc7127d7820", "severity": "medium", "tags": [ - "APM", - "Elastic" + "Elastic", + "APM" ], "type": "query", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/c2_network_connection_from_windows_binary.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/c2_network_connection_from_windows_binary.json deleted file mode 100644 index 5652f025952d7f..00000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/c2_network_connection_from_windows_binary.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "author": [ - "Elastic" - ], - "description": "Identifies network activity from unexpected system applications. This may indicate adversarial activity as these applications are often leveraged by adversaries to execute code and evade detection.", - "from": "now-9m", - "index": [ - "logs-endpoint.events.*", - "winlogbeat-*" - ], - "language": "eql", - "license": "Elastic License", - "name": "Unusual Network Activity from a Windows System Binary", - "query": "sequence by process.entity_id with maxspan=5m\n [process where event.type in (\"start\", \"process_started\") and\n\n /* known applocker bypasses */\n process.name in (\"bginfo.exe\",\n \"cdb.exe\",\n \"control.exe\",\n \"cmstp.exe\",\n \"csi.exe\",\n \"dnx.exe\",\n \"fsi.exe\",\n \"ieexec.exe\",\n \"iexpress.exe\",\n \"installutil.exe\",\n \"Microsoft.Workflow.Compiler.exe\",\n \"MSBuild.exe\",\n \"msdt.exe\",\n \"mshta.exe\",\n \"msiexec.exe\",\n \"msxsl.exe\",\n \"odbcconf.exe\",\n \"rcsi.exe\",\n \"regsvr32.exe\",\n \"xwizard.exe\")]\n [network where event.type == \"connection\" and\n process.name in (\"bginfo.exe\",\n \"cdb.exe\",\n \"control.exe\",\n \"cmstp.exe\",\n \"csi.exe\",\n \"dnx.exe\",\n \"fsi.exe\",\n \"ieexec.exe\",\n \"iexpress.exe\",\n \"installutil.exe\",\n \"Microsoft.Workflow.Compiler.exe\",\n \"MSBuild.exe\",\n \"msdt.exe\",\n \"mshta.exe\",\n \"msiexec.exe\",\n \"msxsl.exe\",\n \"odbcconf.exe\",\n \"rcsi.exe\",\n \"regsvr32.exe\",\n \"xwizard.exe\")]\n", - "risk_score": 21, - "rule_id": "1fe3b299-fbb5-4657-a937-1d746f2c711a", - "severity": "medium", - "tags": [ - "Elastic", - "Windows" - ], - "threat": [ - { - "framework": "MITRE ATT&CK", - "tactic": { - "id": "TA0005", - "name": "Defense Evasion", - "reference": "https://attack.mitre.org/tactics/TA0005/" - }, - "technique": [ - { - "id": "T1127", - "name": "Trusted Developer Utilities Proxy Execution", - "reference": "https://attack.mitre.org/techniques/T1127/" - } - ] - } - ], - "type": "eql", - "version": 1 -} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/collection_cloudtrail_logging_created.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/collection_cloudtrail_logging_created.json index 3dc084a3af54b3..7c98f47fd05e2c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/collection_cloudtrail_logging_created.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/collection_cloudtrail_logging_created.json @@ -25,11 +25,12 @@ "rule_id": "594e0cbf-86cc-45aa-9ff7-ff27db27d3ed", "severity": "low", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Logging", - "Continuous Monitoring" + "Log Auditing" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/collection_gcp_pub_sub_subscription_creation.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/collection_gcp_pub_sub_subscription_creation.json index 720c6f71dafddc..6b90ec776926c5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/collection_gcp_pub_sub_subscription_creation.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/collection_gcp_pub_sub_subscription_creation.json @@ -22,10 +22,11 @@ "severity": "low", "tags": [ "Elastic", + "Cloud", "GCP", "Continuous Monitoring", "SecOps", - "Logging" + "Log Auditing" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/collection_gcp_pub_sub_topic_creation.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/collection_gcp_pub_sub_topic_creation.json index 93695334faae20..e53c36b2366394 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/collection_gcp_pub_sub_topic_creation.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/collection_gcp_pub_sub_topic_creation.json @@ -22,10 +22,11 @@ "severity": "low", "tags": [ "Elastic", + "Cloud", "GCP", "Continuous Monitoring", "SecOps", - "Logging" + "Log Auditing" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/collection_update_event_hub_auth_rule.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/collection_update_event_hub_auth_rule.json index cddc98ba2e6d79..d65a0bcdbc6d0e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/collection_update_event_hub_auth_rule.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/collection_update_event_hub_auth_rule.json @@ -22,11 +22,12 @@ "rule_id": "b6dce542-2b75-4ffb-b7d6-38787298ba9d", "severity": "medium", "tags": [ - "Azure", "Elastic", - "SecOps", + "Cloud", + "Azure", "Continuous Monitoring", - "Logging" + "SecOps", + "Log Auditing" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_certutil_network_connection.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_certutil_network_connection.json index f32877da78d99a..e88297aa2c813f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_certutil_network_connection.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_certutil_network_connection.json @@ -17,7 +17,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Command and Control" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_cobalt_strike_beacon.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_cobalt_strike_beacon.json index 7ebc13ac8079bf..27ad410df1fa24 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_cobalt_strike_beacon.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_cobalt_strike_beacon.json @@ -23,7 +23,9 @@ "severity": "high", "tags": [ "Elastic", - "Network" + "Network", + "Threat Detection", + "Command and Control" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_dns_directly_to_the_internet.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_dns_directly_to_the_internet.json index 79ec202c41ffb6..3df567b09055aa 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_dns_directly_to_the_internet.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_dns_directly_to_the_internet.json @@ -23,7 +23,9 @@ "severity": "medium", "tags": [ "Elastic", - "Network" + "Network", + "Threat Detection", + "Command and Control" ], "threat": [ { @@ -43,5 +45,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_download_rar_powershell_from_internet.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_download_rar_powershell_from_internet.json index 7b739f005a0cbd..92411011ba66e8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_download_rar_powershell_from_internet.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_download_rar_powershell_from_internet.json @@ -23,7 +23,9 @@ "severity": "medium", "tags": [ "Elastic", - "Network" + "Network", + "Threat Detection", + "Command and Control" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_fin7_c2_behavior.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_fin7_c2_behavior.json index 04d68aff0da1c1..1ea40aad7861ae 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_fin7_c2_behavior.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_fin7_c2_behavior.json @@ -22,7 +22,9 @@ "severity": "high", "tags": [ "Elastic", - "Network" + "Network", + "Threat Detection", + "Command and Control" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_ftp_file_transfer_protocol_activity_to_the_internet.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_ftp_file_transfer_protocol_activity_to_the_internet.json index 9a009ffd3fd219..c73fdf1bded9d7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_ftp_file_transfer_protocol_activity_to_the_internet.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_ftp_file_transfer_protocol_activity_to_the_internet.json @@ -8,7 +8,8 @@ ], "index": [ "filebeat-*", - "packetbeat-*" + "packetbeat-*", + "logs-endpoint.events.*" ], "language": "kuery", "license": "Elastic License", @@ -19,7 +20,10 @@ "severity": "low", "tags": [ "Elastic", - "Network" + "Host", + "Network", + "Threat Detection", + "Command and Control" ], "threat": [ { @@ -54,5 +58,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_halfbaked_beacon.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_halfbaked_beacon.json index 7dacb9afcbd60e..19c2832b4b82e3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_halfbaked_beacon.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_halfbaked_beacon.json @@ -23,7 +23,9 @@ "severity": "high", "tags": [ "Elastic", - "Network" + "Network", + "Threat Detection", + "Command and Control" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_irc_internet_relay_chat_protocol_activity_to_the_internet.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_irc_internet_relay_chat_protocol_activity_to_the_internet.json index 0e35d4b1c5ca00..f1901fa70def26 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_irc_internet_relay_chat_protocol_activity_to_the_internet.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_irc_internet_relay_chat_protocol_activity_to_the_internet.json @@ -8,7 +8,8 @@ ], "index": [ "filebeat-*", - "packetbeat-*" + "packetbeat-*", + "logs-endpoint.events.*" ], "language": "kuery", "license": "Elastic License", @@ -19,7 +20,10 @@ "severity": "medium", "tags": [ "Elastic", - "Network" + "Host", + "Network", + "Threat Detection", + "Command and Control" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_nat_traversal_port_activity.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_nat_traversal_port_activity.json index e42bf4029eb018..5afdd1f629ae4a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_nat_traversal_port_activity.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_nat_traversal_port_activity.json @@ -8,7 +8,8 @@ ], "index": [ "filebeat-*", - "packetbeat-*" + "packetbeat-*", + "logs-endpoint.events.*" ], "language": "kuery", "license": "Elastic License", @@ -19,7 +20,10 @@ "severity": "low", "tags": [ "Elastic", - "Network" + "Host", + "Network", + "Threat Detection", + "Command and Control" ], "threat": [ { @@ -39,5 +43,5 @@ } ], "type": "query", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_port_26_activity.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_port_26_activity.json index 1cdfd44eb2adf3..edd913da4d2b36 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_port_26_activity.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_port_26_activity.json @@ -8,7 +8,8 @@ ], "index": [ "filebeat-*", - "packetbeat-*" + "packetbeat-*", + "logs-endpoint.events.*" ], "language": "kuery", "license": "Elastic License", @@ -23,7 +24,10 @@ "severity": "low", "tags": [ "Elastic", - "Network" + "Host", + "Network", + "Threat Detection", + "Command and Control" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_port_8000_activity_to_the_internet.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_port_8000_activity_to_the_internet.json index 319f95ed88e08b..0c35bd5e23ed53 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_port_8000_activity_to_the_internet.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_port_8000_activity_to_the_internet.json @@ -8,7 +8,8 @@ ], "index": [ "filebeat-*", - "packetbeat-*" + "packetbeat-*", + "logs-endpoint.events.*" ], "language": "kuery", "license": "Elastic License", @@ -19,7 +20,10 @@ "severity": "low", "tags": [ "Elastic", - "Network" + "Host", + "Network", + "Threat Detection", + "Command and Control" ], "threat": [ { @@ -39,5 +43,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_pptp_point_to_point_tunneling_protocol_activity.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_pptp_point_to_point_tunneling_protocol_activity.json index bd478f2b23fc03..c706a5b7248c86 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_pptp_point_to_point_tunneling_protocol_activity.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_pptp_point_to_point_tunneling_protocol_activity.json @@ -8,7 +8,8 @@ ], "index": [ "filebeat-*", - "packetbeat-*" + "packetbeat-*", + "logs-endpoint.events.*" ], "language": "kuery", "license": "Elastic License", @@ -19,7 +20,10 @@ "severity": "low", "tags": [ "Elastic", - "Network" + "Host", + "Network", + "Threat Detection", + "Command and Control" ], "threat": [ { @@ -39,5 +43,5 @@ } ], "type": "query", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_proxy_port_activity_to_the_internet.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_proxy_port_activity_to_the_internet.json index ee025053006115..8535a9591b88f2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_proxy_port_activity_to_the_internet.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_proxy_port_activity_to_the_internet.json @@ -8,7 +8,8 @@ ], "index": [ "filebeat-*", - "packetbeat-*" + "packetbeat-*", + "logs-endpoint.events.*" ], "language": "kuery", "license": "Elastic License", @@ -19,7 +20,10 @@ "severity": "medium", "tags": [ "Elastic", - "Network" + "Host", + "Network", + "Threat Detection", + "Command and Control" ], "threat": [ { @@ -39,5 +43,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_rdp_remote_desktop_protocol_from_the_internet.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_rdp_remote_desktop_protocol_from_the_internet.json index 87544647b17e13..4a3fd026f54a76 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_rdp_remote_desktop_protocol_from_the_internet.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_rdp_remote_desktop_protocol_from_the_internet.json @@ -8,7 +8,8 @@ ], "index": [ "filebeat-*", - "packetbeat-*" + "packetbeat-*", + "logs-endpoint.events.*" ], "language": "kuery", "license": "Elastic License", @@ -19,7 +20,10 @@ "severity": "medium", "tags": [ "Elastic", - "Network" + "Host", + "Network", + "Threat Detection", + "Command and Control" ], "threat": [ { @@ -69,5 +73,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_remote_file_copy_desktopimgdownldr.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_remote_file_copy_desktopimgdownldr.json index d5b21dfe2db18f..596c4bbac57bab 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_remote_file_copy_desktopimgdownldr.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_remote_file_copy_desktopimgdownldr.json @@ -11,7 +11,7 @@ "language": "kuery", "license": "Elastic License", "name": "Remote File Download via Desktopimgdownldr Utility", - "query": "event.category:process and event.type:(start or process_started) and (process.name:desktopimgdownldr.exe or process.pe.original_file_name:desktopimgdownldr.exe) and process.args:/lockscreenurl\\:http*", + "query": "event.category:process and event.type:(start or process_started) and (process.name:desktopimgdownldr.exe or process.pe.original_file_name:desktopimgdownldr.exe or winlog.event_data.OriginalFileName:desktopimgdownldr.exe) and process.args:/lockscreenurl\\:http*", "references": [ "https://labs.sentinelone.com/living-off-windows-land-a-new-native-file-downldr/" ], @@ -20,7 +20,10 @@ "severity": "medium", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Command and Control" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_remote_file_copy_mpcmdrun.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_remote_file_copy_mpcmdrun.json index aeadc849eac17b..9eef2fbbc62a6a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_remote_file_copy_mpcmdrun.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_remote_file_copy_mpcmdrun.json @@ -12,7 +12,7 @@ "license": "Elastic License", "name": "Remote File Download via MpCmdRun", "note": "### Investigating Remote File Download via MpCmdRun\nVerify details such as the parent process, URL reputation, and downloaded file details. Additionally, `MpCmdRun` logs this information in the Appdata Temp folder in `MpCmdRun.log`.", - "query": "event.category:process and event.type:(start or process_started) and (process.name:MpCmdRun.exe or process.pe.original_file_name:MpCmdRun.exe) and process.args:((\"-DownloadFile\" or \"-downloadfile\") and \"-url\" and \"-path\")", + "query": "event.category:process and event.type:(start or process_started) and (process.name:MpCmdRun.exe or process.pe.original_file_name:MpCmdRun.exe or winlog.event_data.OriginalFileName:MpCmdRun.exe) and process.args:((\"-DownloadFile\" or \"-downloadfile\") and \"-url\" and \"-path\")", "references": [ "https://twitter.com/mohammadaskar2/status/1301263551638761477", "https://www.bleepingcomputer.com/news/microsoft/microsoft-defender-can-ironically-be-used-to-download-malware/" @@ -22,7 +22,10 @@ "severity": "medium", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Command and Control" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_smtp_to_the_internet.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_smtp_to_the_internet.json index 3a082c29a4cf1d..f041255374f12c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_smtp_to_the_internet.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_smtp_to_the_internet.json @@ -8,7 +8,8 @@ ], "index": [ "filebeat-*", - "packetbeat-*" + "packetbeat-*", + "logs-endpoint.events.*" ], "language": "kuery", "license": "Elastic License", @@ -19,7 +20,10 @@ "severity": "low", "tags": [ "Elastic", - "Network" + "Host", + "Network", + "Threat Detection", + "Command and Control" ], "threat": [ { @@ -54,5 +58,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_sql_server_port_activity_to_the_internet.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_sql_server_port_activity_to_the_internet.json index 95ac4d88368007..7e4f3907fc31ea 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_sql_server_port_activity_to_the_internet.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_sql_server_port_activity_to_the_internet.json @@ -8,7 +8,8 @@ ], "index": [ "filebeat-*", - "packetbeat-*" + "packetbeat-*", + "logs-endpoint.events.*" ], "language": "kuery", "license": "Elastic License", @@ -19,7 +20,10 @@ "severity": "medium", "tags": [ "Elastic", - "Network" + "Host", + "Network", + "Threat Detection", + "Command and Control" ], "threat": [ { @@ -39,5 +43,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_ssh_secure_shell_from_the_internet.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_ssh_secure_shell_from_the_internet.json index fe5608459ffce7..08ab14aeb5c7ca 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_ssh_secure_shell_from_the_internet.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_ssh_secure_shell_from_the_internet.json @@ -8,7 +8,8 @@ ], "index": [ "filebeat-*", - "packetbeat-*" + "packetbeat-*", + "logs-endpoint.events.*" ], "language": "kuery", "license": "Elastic License", @@ -19,7 +20,10 @@ "severity": "medium", "tags": [ "Elastic", - "Network" + "Host", + "Network", + "Threat Detection", + "Command and Control" ], "threat": [ { @@ -69,5 +73,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_ssh_secure_shell_to_the_internet.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_ssh_secure_shell_to_the_internet.json index 9ecfe39a793032..4bc48ebe0c3160 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_ssh_secure_shell_to_the_internet.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_ssh_secure_shell_to_the_internet.json @@ -8,7 +8,8 @@ ], "index": [ "filebeat-*", - "packetbeat-*" + "packetbeat-*", + "logs-endpoint.events.*" ], "language": "kuery", "license": "Elastic License", @@ -19,7 +20,10 @@ "severity": "low", "tags": [ "Elastic", - "Network" + "Host", + "Network", + "Threat Detection", + "Command and Control" ], "threat": [ { @@ -39,5 +43,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_teamviewer_remote_file_copy.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_teamviewer_remote_file_copy.json index 793ff4ebda72f8..c597f6b2718b9a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_teamviewer_remote_file_copy.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_teamviewer_remote_file_copy.json @@ -20,7 +20,10 @@ "severity": "medium", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Command and Control" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_telnet_port_activity.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_telnet_port_activity.json index 4455d8adfdf837..9007db322ae580 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_telnet_port_activity.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_telnet_port_activity.json @@ -8,7 +8,8 @@ ], "index": [ "filebeat-*", - "packetbeat-*" + "packetbeat-*", + "logs-endpoint.events.*" ], "language": "kuery", "license": "Elastic License", @@ -19,7 +20,10 @@ "severity": "medium", "tags": [ "Elastic", - "Network" + "Host", + "Network", + "Threat Detection", + "Command and Control" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_tor_activity_to_the_internet.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_tor_activity_to_the_internet.json index b278c36d01c1b7..e82106a87bc2e1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_tor_activity_to_the_internet.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_tor_activity_to_the_internet.json @@ -8,7 +8,8 @@ ], "index": [ "filebeat-*", - "packetbeat-*" + "packetbeat-*", + "logs-endpoint.events.*" ], "language": "kuery", "license": "Elastic License", @@ -19,7 +20,10 @@ "severity": "medium", "tags": [ "Elastic", - "Network" + "Host", + "Network", + "Threat Detection", + "Command and Control" ], "threat": [ { @@ -54,5 +58,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_vnc_virtual_network_computing_from_the_internet.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_vnc_virtual_network_computing_from_the_internet.json index 97d2b940a69498..9321d2a2103de5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_vnc_virtual_network_computing_from_the_internet.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_vnc_virtual_network_computing_from_the_internet.json @@ -8,7 +8,8 @@ ], "index": [ "filebeat-*", - "packetbeat-*" + "packetbeat-*", + "logs-endpoint.events.*" ], "language": "kuery", "license": "Elastic License", @@ -19,7 +20,10 @@ "severity": "high", "tags": [ "Elastic", - "Network" + "Host", + "Network", + "Threat Detection", + "Command and Control" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_vnc_virtual_network_computing_to_the_internet.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_vnc_virtual_network_computing_to_the_internet.json index 97757af22be0c2..38f38e97626453 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_vnc_virtual_network_computing_to_the_internet.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/command_and_control_vnc_virtual_network_computing_to_the_internet.json @@ -8,7 +8,8 @@ ], "index": [ "filebeat-*", - "packetbeat-*" + "packetbeat-*", + "logs-endpoint.events.*" ], "language": "kuery", "license": "Elastic License", @@ -19,7 +20,10 @@ "severity": "medium", "tags": [ "Elastic", - "Network" + "Host", + "Network", + "Threat Detection", + "Command and Control" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_attempted_bypass_of_okta_mfa.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_attempted_bypass_of_okta_mfa.json index 118f8f6b2ad4aa..fb8256bf2509c4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_attempted_bypass_of_okta_mfa.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_attempted_bypass_of_okta_mfa.json @@ -21,10 +21,11 @@ "severity": "high", "tags": [ "Elastic", + "Identity", "Okta", + "Continuous Monitoring", "SecOps", - "Identity and Access", - "Continuous Monitoring" + "Identity and Access" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_attempts_to_brute_force_okta_user_account.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_attempts_to_brute_force_okta_user_account.json index 5aae95476e9dad..d8d5b5305aaaab 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_attempts_to_brute_force_okta_user_account.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_attempts_to_brute_force_okta_user_account.json @@ -22,6 +22,7 @@ "severity": "medium", "tags": [ "Elastic", + "Identity", "Okta", "Continuous Monitoring", "SecOps", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_aws_iam_assume_role_brute_force.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_aws_iam_assume_role_brute_force.json index e350c3697f6852..b7e89a9120bbb1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_aws_iam_assume_role_brute_force.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_aws_iam_assume_role_brute_force.json @@ -21,11 +21,12 @@ "rule_id": "ea248a02-bc47-4043-8e94-2885b19b2636", "severity": "medium", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Identity and Access", - "Continuous Monitoring" + "Identity and Access" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/exfiltration_compress_credentials_keychains.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_compress_credentials_keychains.json similarity index 95% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/exfiltration_compress_credentials_keychains.json rename to x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_compress_credentials_keychains.json index bf2a52066ae1c0..c13ac69e509870 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/exfiltration_compress_credentials_keychains.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_compress_credentials_keychains.json @@ -20,7 +20,10 @@ "severity": "high", "tags": [ "Elastic", - "MacOS" + "Host", + "macOS", + "Threat Detection", + "Credential Access" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_credential_dumping_msbuild.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_credential_dumping_msbuild.json index a67fa01ab371a2..0761ba515d9b1d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_credential_dumping_msbuild.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_credential_dumping_msbuild.json @@ -20,7 +20,10 @@ "severity": "high", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Credential Access" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_domain_backup_dpapi_private_keys.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_domain_backup_dpapi_private_keys.json index dc4f5e11754d30..eefd6ee9e601b8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_domain_backup_dpapi_private_keys.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_domain_backup_dpapi_private_keys.json @@ -22,7 +22,10 @@ "severity": "high", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Credential Access" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_gcp_iam_service_account_key_deletion.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_gcp_iam_service_account_key_deletion.json index 63d5081869f1b1..5db891caa28577 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_gcp_iam_service_account_key_deletion.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_gcp_iam_service_account_key_deletion.json @@ -23,6 +23,7 @@ "severity": "low", "tags": [ "Elastic", + "Cloud", "GCP", "Continuous Monitoring", "SecOps", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_gcp_key_created_for_service_account.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_gcp_key_created_for_service_account.json index c1ae7f5fc1953e..a6d45b74657717 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_gcp_key_created_for_service_account.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_gcp_key_created_for_service_account.json @@ -23,6 +23,7 @@ "severity": "low", "tags": [ "Elastic", + "Cloud", "GCP", "Continuous Monitoring", "SecOps", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_iam_user_addition_to_group.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_iam_user_addition_to_group.json index 7c5aa9bc7f3a8b..8244cb755787f8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_iam_user_addition_to_group.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_iam_user_addition_to_group.json @@ -24,11 +24,12 @@ "rule_id": "333de828-8190-4cf5-8d7c-7575846f6fe0", "severity": "low", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Identity and Access", - "Continuous Monitoring" + "Identity and Access" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_iis_apppoolsa_pwd_appcmd.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_iis_apppoolsa_pwd_appcmd.json index dd7bc43c583826..6a182617945f1b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_iis_apppoolsa_pwd_appcmd.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_iis_apppoolsa_pwd_appcmd.json @@ -12,7 +12,7 @@ "license": "Elastic License", "max_signals": 33, "name": "Microsoft IIS Service Account Password Dumped", - "query": "event.category:process AND event.type:(start OR process_started) AND (process.name:appcmd.exe OR process.pe.original_file_name:appcmd.exe) AND process.args:(/[lL][iI][sS][tT]/ AND /\\/[tT][eE][xX][tT]\\:[pP][aA][sS][sS][wW][oO][rR][dD]/)", + "query": "event.category:process AND event.type:(start OR process_started) AND (process.name:appcmd.exe OR process.pe.original_file_name:appcmd.exe or winlog.event_data.OriginalFileName:appcmd.exe) AND process.args:(/[lL][iI][sS][tT]/ AND /\\/[tT][eE][xX][tT]\\:[pP][aA][sS][sS][wW][oO][rR][dD]/)", "references": [ "https://blog.netspi.com/decrypting-iis-passwords-to-break-out-of-the-dmz-part-1/" ], @@ -21,7 +21,10 @@ "severity": "high", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Credential Access" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_iis_connectionstrings_dumping.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_iis_connectionstrings_dumping.json index 2735fcbbd6130f..f750a0f5594b47 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_iis_connectionstrings_dumping.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_iis_connectionstrings_dumping.json @@ -12,7 +12,7 @@ "license": "Elastic License", "max_signals": 33, "name": "Microsoft IIS Connection Strings Decryption", - "query": "event.category:process and event.type:(start or process_started) and (process.name:aspnet_regiis.exe or process.pe.original_file_name:aspnet_regiis.exe) and process.args:(connectionStrings and \"-pdf\")", + "query": "event.category:process and event.type:(start or process_started) and (process.name:aspnet_regiis.exe or process.pe.original_file_name:aspnet_regiis.exe or winlog.event_data.OriginalFileName:aspnet_regiis.exe) and process.args:(connectionStrings and \"-pdf\")", "references": [ "https://blog.netspi.com/decrypting-iis-passwords-to-break-out-of-the-dmz-part-1/", "https://symantec-enterprise-blogs.security.com/blogs/threat-intelligence/greenbug-espionage-telco-south-asia" @@ -22,7 +22,10 @@ "severity": "high", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Credential Access" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_kerberosdump_kcc.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_kerberosdump_kcc.json index 4713d09f8adec0..dda2f67c116632 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_kerberosdump_kcc.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_kerberosdump_kcc.json @@ -21,7 +21,10 @@ "severity": "high", "tags": [ "Elastic", - "MacOS" + "Host", + "macOS", + "Threat Detection", + "Credential Access" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_key_vault_modified.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_key_vault_modified.json index a45591c73dcb35..33df4e59300662 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_key_vault_modified.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_key_vault_modified.json @@ -24,9 +24,10 @@ "severity": "medium", "tags": [ "Elastic", + "Cloud", "Azure", - "SecOps", "Continuous Monitoring", + "SecOps", "Data Protection" ], "threat": [ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_mimikatz_memssp_default_logs.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_mimikatz_memssp_default_logs.json index fa1f99eef7f001..ae9b63371a4524 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_mimikatz_memssp_default_logs.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_mimikatz_memssp_default_logs.json @@ -17,7 +17,10 @@ "severity": "high", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Credential Access" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_okta_brute_force_or_password_spraying.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_okta_brute_force_or_password_spraying.json index c36f878792ccfa..9e10dd6dae522b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_okta_brute_force_or_password_spraying.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_okta_brute_force_or_password_spraying.json @@ -24,10 +24,11 @@ "severity": "medium", "tags": [ "Elastic", + "Identity", "Okta", + "Continuous Monitoring", "SecOps", - "Identity and Access", - "Continuous Monitoring" + "Identity and Access" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_root_console_failure_brute_force.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_root_console_failure_brute_force.json similarity index 96% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_root_console_failure_brute_force.json rename to x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_root_console_failure_brute_force.json index 5f7781be82efdd..f65a7b11d11ad6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_root_console_failure_brute_force.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_root_console_failure_brute_force.json @@ -23,11 +23,12 @@ "rule_id": "4d50a94f-2844-43fa-8395-6afbd5e1c5ef", "severity": "high", "tags": [ + "Elastic", + "Cloud", "AWS", "Continuous Monitoring", - "Elastic", - "Identity and Access", - "SecOps" + "SecOps", + "Identity and Access" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_secretsmanager_getsecretvalue.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_secretsmanager_getsecretvalue.json index 879e93750df9cf..a33593f701ba7b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_secretsmanager_getsecretvalue.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_secretsmanager_getsecretvalue.json @@ -26,11 +26,12 @@ "rule_id": "a00681e3-9ed6-447c-ab2c-be648821c622", "severity": "high", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Data Protection", - "Continuous Monitoring" + "Data Protection" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_storage_account_key_regenerated.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_storage_account_key_regenerated.json index 2a3dc85294a9db..62e1aab700680a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_storage_account_key_regenerated.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_storage_account_key_regenerated.json @@ -22,10 +22,11 @@ "rule_id": "1e0b832e-957e-43ae-b319-db82d228c908", "severity": "low", "tags": [ - "Azure", "Elastic", - "SecOps", + "Cloud", + "Azure", "Continuous Monitoring", + "SecOps", "Identity and Access" ], "threat": [ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_tcpdump_activity.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_tcpdump_activity.json index 306a38f5d2a28f..67ab62e8d0ee53 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_tcpdump_activity.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_tcpdump_activity.json @@ -20,7 +20,10 @@ "severity": "low", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection", + "Credential Access" ], "threat": [ { @@ -55,5 +58,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_adding_the_hidden_file_attribute_with_via_attribexe.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_adding_the_hidden_file_attribute_with_via_attribexe.json index c80f24a21d958d..a1ff4bfc890a11 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_adding_the_hidden_file_attribute_with_via_attribexe.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_adding_the_hidden_file_attribute_with_via_attribexe.json @@ -17,7 +17,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { @@ -52,5 +55,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_attempt_to_disable_iptables_or_firewall.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_attempt_to_disable_iptables_or_firewall.json index 4d4f10bbaa5997..b17e4979a885c6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_attempt_to_disable_iptables_or_firewall.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_attempt_to_disable_iptables_or_firewall.json @@ -17,7 +17,10 @@ "severity": "medium", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection", + "Defense Evasion" ], "threat": [ { @@ -37,5 +40,5 @@ } ], "type": "query", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_attempt_to_disable_syslog_service.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_attempt_to_disable_syslog_service.json index 3c34b04a77a500..960000c91e4fa1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_attempt_to_disable_syslog_service.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_attempt_to_disable_syslog_service.json @@ -17,7 +17,10 @@ "severity": "medium", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection", + "Defense Evasion" ], "threat": [ { @@ -37,5 +40,5 @@ } ], "type": "query", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_azure_diagnostic_settings_deletion.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_azure_diagnostic_settings_deletion.json index 49d98813dc0409..7721790b5cf975 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_azure_diagnostic_settings_deletion.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_azure_diagnostic_settings_deletion.json @@ -23,9 +23,10 @@ "severity": "medium", "tags": [ "Elastic", + "Cloud", "Azure", - "SecOps", "Continuous Monitoring", + "SecOps", "Monitoring" ], "threat": [ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_base16_or_base32_encoding_or_decoding_activity.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_base16_or_base32_encoding_or_decoding_activity.json index 3cdfac92572b19..140e1ccd8e8905 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_base16_or_base32_encoding_or_decoding_activity.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_base16_or_base32_encoding_or_decoding_activity.json @@ -20,7 +20,10 @@ "severity": "low", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection", + "Defense Evasion" ], "threat": [ { @@ -55,5 +58,5 @@ } ], "type": "query", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_base64_encoding_or_decoding_activity.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_base64_encoding_or_decoding_activity.json index 2d26d867b8718e..fa322fca5db8ad 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_base64_encoding_or_decoding_activity.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_base64_encoding_or_decoding_activity.json @@ -20,7 +20,10 @@ "severity": "low", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection", + "Defense Evasion" ], "threat": [ { @@ -55,5 +58,5 @@ } ], "type": "query", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_clearing_windows_event_logs.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_clearing_windows_event_logs.json index 60ce575148f4c1..11d57b855f974a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_clearing_windows_event_logs.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_clearing_windows_event_logs.json @@ -17,7 +17,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { @@ -37,5 +40,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_cloudtrail_logging_deleted.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_cloudtrail_logging_deleted.json index 64261af2a31052..169f429a6dd266 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_cloudtrail_logging_deleted.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_cloudtrail_logging_deleted.json @@ -25,11 +25,12 @@ "rule_id": "7024e2a0-315d-4334-bb1a-441c593e16ab", "severity": "medium", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Logging", - "Continuous Monitoring" + "Log Auditing" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_cloudtrail_logging_suspended.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_cloudtrail_logging_suspended.json index 090073698026d2..cbd040a7f7a30d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_cloudtrail_logging_suspended.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_cloudtrail_logging_suspended.json @@ -25,11 +25,12 @@ "rule_id": "1aa8fa52-44a7-4dae-b058-f3333b91c8d7", "severity": "medium", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Logging", - "Continuous Monitoring" + "Log Auditing" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_cloudwatch_alarm_deletion.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_cloudwatch_alarm_deletion.json index aeaf0a41688140..e18deb65c497bc 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_cloudwatch_alarm_deletion.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_cloudwatch_alarm_deletion.json @@ -25,11 +25,12 @@ "rule_id": "f772ec8a-e182-483c-91d2-72058f76a44c", "severity": "medium", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Monitoring", - "Continuous Monitoring" + "Monitoring" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_code_injection_conhost.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_code_injection_conhost.json index 63c7ea12b3b6bd..ed1b8978c63299 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_code_injection_conhost.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_code_injection_conhost.json @@ -21,7 +21,10 @@ "severity": "high", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_config_service_rule_deletion.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_config_service_rule_deletion.json index 268f52a8efd5e0..b7d9321814fd79 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_config_service_rule_deletion.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_config_service_rule_deletion.json @@ -25,11 +25,12 @@ "rule_id": "7024e2a0-315d-4334-bb1a-552d604f27bc", "severity": "medium", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Monitoring", - "Continuous Monitoring" + "Monitoring" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_configuration_recorder_stopped.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_configuration_recorder_stopped.json index b926937450f5a1..b28572deaf2047 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_configuration_recorder_stopped.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_configuration_recorder_stopped.json @@ -25,11 +25,12 @@ "rule_id": "fbd44836-0d69-4004-a0b4-03c20370c435", "severity": "high", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Monitoring", - "Continuous Monitoring" + "Monitoring" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_cve_2020_0601.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_cve_2020_0601.json index 2abad3c255f154..3beb71763f1ae6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_cve_2020_0601.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_cve_2020_0601.json @@ -15,7 +15,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { @@ -35,5 +38,5 @@ } ], "type": "query", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_delete_volume_usn_journal_with_fsutil.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_delete_volume_usn_journal_with_fsutil.json index 50213b9f1a42c4..5fde3c462ededf 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_delete_volume_usn_journal_with_fsutil.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_delete_volume_usn_journal_with_fsutil.json @@ -17,7 +17,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { @@ -37,5 +40,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_deleting_backup_catalogs_with_wbadmin.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_deleting_backup_catalogs_with_wbadmin.json index 026735f413eabd..554ccc6972e5d3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_deleting_backup_catalogs_with_wbadmin.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_deleting_backup_catalogs_with_wbadmin.json @@ -17,7 +17,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { @@ -37,5 +40,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_deletion_of_bash_command_line_history.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_deletion_of_bash_command_line_history.json index 85d8bdcb2582f4..eef37499c8eb5f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_deletion_of_bash_command_line_history.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_deletion_of_bash_command_line_history.json @@ -17,7 +17,10 @@ "severity": "medium", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection", + "Defense Evasion" ], "threat": [ { @@ -37,5 +40,5 @@ } ], "type": "query", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_disable_selinux_attempt.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_disable_selinux_attempt.json index d107c0b2620919..35476a76fd4b50 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_disable_selinux_attempt.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_disable_selinux_attempt.json @@ -17,7 +17,10 @@ "severity": "medium", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection", + "Defense Evasion" ], "threat": [ { @@ -37,5 +40,5 @@ } ], "type": "query", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_disable_windows_firewall_rules_with_netsh.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_disable_windows_firewall_rules_with_netsh.json index 6fbf9ca800f795..a69fde9f6a5cc7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_disable_windows_firewall_rules_with_netsh.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_disable_windows_firewall_rules_with_netsh.json @@ -17,7 +17,10 @@ "severity": "medium", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { @@ -37,5 +40,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_suspicious_dotnet_compiler_parent_process.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_dotnet_compiler_parent_process.json similarity index 93% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_suspicious_dotnet_compiler_parent_process.json rename to x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_dotnet_compiler_parent_process.json index 0a675bd7aab748..08cbb33710b264 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_suspicious_dotnet_compiler_parent_process.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_dotnet_compiler_parent_process.json @@ -17,7 +17,10 @@ "severity": "medium", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_ec2_flow_log_deletion.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_ec2_flow_log_deletion.json index 9f3d4e6b5e3790..d36294684698e0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_ec2_flow_log_deletion.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_ec2_flow_log_deletion.json @@ -25,11 +25,12 @@ "rule_id": "9395fd2c-9947-4472-86ef-4aceb2f7e872", "severity": "high", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Logging", - "Continuous Monitoring" + "Log Auditing" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_ec2_network_acl_deletion.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_ec2_network_acl_deletion.json index 6ecc9ad3d558df..b6ac9be8008072 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_ec2_network_acl_deletion.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_ec2_network_acl_deletion.json @@ -27,11 +27,12 @@ "rule_id": "8623535c-1e17-44e1-aa97-7a0699c3037d", "severity": "medium", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Network", - "Continuous Monitoring" + "Network Security" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_encoding_or_decoding_files_via_certutil.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_encoding_or_decoding_files_via_certutil.json index 0d47aab2c64bde..f00ac3f90527ba 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_encoding_or_decoding_files_via_certutil.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_encoding_or_decoding_files_via_certutil.json @@ -17,7 +17,10 @@ "severity": "medium", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { @@ -37,5 +40,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_event_hub_deletion.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_event_hub_deletion.json index 29df07cced4d7d..d09edf473c939c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_event_hub_deletion.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_event_hub_deletion.json @@ -24,11 +24,12 @@ "rule_id": "e0f36de1-0342-453d-95a9-a068b257b053", "severity": "medium", "tags": [ - "Azure", "Elastic", - "SecOps", + "Cloud", + "Azure", "Continuous Monitoring", - "Logging" + "SecOps", + "Log Auditing" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_msbuild_started_by_office_app.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_msbuild_started_by_office_app.json index a987c00b392eca..c41b3aad2f42cb 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_msbuild_started_by_office_app.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_msbuild_started_by_office_app.json @@ -23,7 +23,10 @@ "severity": "high", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_msbuild_started_by_script.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_msbuild_started_by_script.json index 0537f27bad4636..a8e16d8bda2383 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_msbuild_started_by_script.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_msbuild_started_by_script.json @@ -20,7 +20,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_msbuild_started_by_system_process.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_msbuild_started_by_system_process.json index 11fdd128475dc1..60c9a317bbb6c0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_msbuild_started_by_system_process.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_msbuild_started_by_system_process.json @@ -20,7 +20,10 @@ "severity": "medium", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_msbuild_started_renamed.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_msbuild_started_renamed.json index a90e5ebc57800c..5daab573db5bd0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_msbuild_started_renamed.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_msbuild_started_renamed.json @@ -20,7 +20,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_msbuild_started_unusal_process.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_msbuild_started_unusal_process.json index 4f353a6ff9e6f1..09247d2f213234 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_msbuild_started_unusal_process.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_msbuild_started_unusal_process.json @@ -23,7 +23,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { @@ -43,5 +46,5 @@ } ], "type": "query", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_suspicious_explorer_winword.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_suspicious_explorer_winword.json index 69d334cf13fdbf..7d9f190ba7be2f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_suspicious_explorer_winword.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_suspicious_explorer_winword.json @@ -11,13 +11,16 @@ "language": "kuery", "license": "Elastic License", "name": "Potential DLL SideLoading via Trusted Microsoft Programs", - "query": "event.category:process and event.type:(start or process_started) and process.pe.original_file_name:(WinWord.exe or EXPLORER.EXE or w3wp.exe or DISM.EXE) and not (process.name:(winword.exe or WINWORD.EXE or explorer.exe or w3wp.exe or Dism.exe) or process.executable:(\"C:\\Windows\\explorer.exe\" or C\\:\\\\Program?Files\\\\Microsoft?Office\\\\root\\\\Office*\\\\WINWORD.EXE or C\\:\\\\Program?Files?\\(x86\\)\\\\Microsoft?Office\\\\root\\\\Office*\\\\WINWORD.EXE or \"C:\\Windows\\System32\\Dism.exe\" or \"C:\\Windows\\SysWOW64\\Dism.exe\" or \"C:\\Windows\\System32\\inetsrv\\w3wp.exe\"))", + "query": "event.category:process and event.type:(start or process_started) and (process.pe.original_file_name:(WinWord.exe or EXPLORER.EXE or w3wp.exe or DISM.EXE) or winlog.event_data.OriginalFileName:(WinWord.exe or EXPLORER.EXE or w3wp.exe or DISM.EXE)) and not (process.name:(winword.exe or WINWORD.EXE or explorer.exe or w3wp.exe or Dism.exe) or process.executable:(\"C:\\Windows\\explorer.exe\" or C\\:\\\\Program?Files\\\\Microsoft?Office\\\\root\\\\Office*\\\\WINWORD.EXE or C\\:\\\\Program?Files?\\(x86\\)\\\\Microsoft?Office\\\\root\\\\Office*\\\\WINWORD.EXE or \"C:\\Windows\\System32\\Dism.exe\" or \"C:\\Windows\\SysWOW64\\Dism.exe\" or \"C:\\Windows\\System32\\inetsrv\\w3wp.exe\"))", "risk_score": 73, "rule_id": "1160dcdb-0a0a-4a79-91d8-9b84616edebd", "severity": "high", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_via_trusted_developer_utilities.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_via_trusted_developer_utilities.json index a1d14155cc3b30..7963c03699f78a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_via_trusted_developer_utilities.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_via_trusted_developer_utilities.json @@ -7,7 +7,8 @@ "These programs may be used by Windows developers but use by non-engineers is unusual." ], "index": [ - "winlogbeat-*" + "winlogbeat-*", + "logs-endpoint.events.*" ], "language": "kuery", "license": "Elastic License", @@ -18,7 +19,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_file_deletion_via_shred.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_file_deletion_via_shred.json index 5b02f63a1c7f70..dc73b7bc1eb767 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_file_deletion_via_shred.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_file_deletion_via_shred.json @@ -17,7 +17,10 @@ "severity": "low", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection", + "Defense Evasion" ], "threat": [ { @@ -37,5 +40,5 @@ } ], "type": "query", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_file_mod_writable_dir.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_file_mod_writable_dir.json index 8ee2d4fda7bf84..b97e5c1d6a1826 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_file_mod_writable_dir.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_file_mod_writable_dir.json @@ -20,7 +20,10 @@ "severity": "low", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection", + "Defense Evasion" ], "threat": [ { @@ -40,5 +43,5 @@ } ], "type": "query", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_firewall_policy_deletion.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_firewall_policy_deletion.json index 759fc9d5ecb1f6..69a123ba678fd6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_firewall_policy_deletion.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_firewall_policy_deletion.json @@ -22,11 +22,12 @@ "rule_id": "e02bd3ea-72c6-4181-ac2b-0f83d17ad969", "severity": "low", "tags": [ - "Azure", "Elastic", - "SecOps", + "Cloud", + "Azure", "Continuous Monitoring", - "Network" + "SecOps", + "Network Security" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_firewall_rule_created.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_firewall_rule_created.json index b80a5f0e179490..dc08dace20bfcd 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_firewall_rule_created.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_firewall_rule_created.json @@ -22,6 +22,7 @@ "severity": "low", "tags": [ "Elastic", + "Cloud", "GCP", "Continuous Monitoring", "SecOps", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_firewall_rule_deleted.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_firewall_rule_deleted.json index 64c8d01df47e95..7ee5af109f37ba 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_firewall_rule_deleted.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_firewall_rule_deleted.json @@ -22,6 +22,7 @@ "severity": "medium", "tags": [ "Elastic", + "Cloud", "GCP", "Continuous Monitoring", "SecOps", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_firewall_rule_modified.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_firewall_rule_modified.json index b2c0e259b45e0b..b4107fb9f08fd7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_firewall_rule_modified.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_firewall_rule_modified.json @@ -22,6 +22,7 @@ "severity": "medium", "tags": [ "Elastic", + "Cloud", "GCP", "Continuous Monitoring", "SecOps", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_logging_bucket_deletion.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_logging_bucket_deletion.json index 62447b789d6321..079a87b5c615b5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_logging_bucket_deletion.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_logging_bucket_deletion.json @@ -23,10 +23,11 @@ "severity": "medium", "tags": [ "Elastic", + "Cloud", "GCP", "Continuous Monitoring", "SecOps", - "Logging" + "Log Auditing" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_logging_sink_deletion.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_logging_sink_deletion.json index 0fc83070ffbb70..8466b618fab986 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_logging_sink_deletion.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_logging_sink_deletion.json @@ -22,10 +22,11 @@ "severity": "medium", "tags": [ "Elastic", + "Cloud", "GCP", "Continuous Monitoring", "SecOps", - "Logging" + "Log Auditing" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_pub_sub_subscription_deletion.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_pub_sub_subscription_deletion.json index 2ae47140b66a5d..5b87b8722595c7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_pub_sub_subscription_deletion.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_pub_sub_subscription_deletion.json @@ -22,10 +22,11 @@ "severity": "low", "tags": [ "Elastic", + "Cloud", "GCP", "Continuous Monitoring", "SecOps", - "Logging" + "Log Auditing" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_pub_sub_topic_deletion.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_pub_sub_topic_deletion.json index f276af3e21862b..5a681a35006a78 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_pub_sub_topic_deletion.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_pub_sub_topic_deletion.json @@ -22,10 +22,11 @@ "severity": "low", "tags": [ "Elastic", + "Cloud", "GCP", "Continuous Monitoring", "SecOps", - "Logging" + "Log Auditing" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_storage_bucket_configuration_modified.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_storage_bucket_configuration_modified.json index 3b18732137c322..5992beef9873e8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_storage_bucket_configuration_modified.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_storage_bucket_configuration_modified.json @@ -22,6 +22,7 @@ "severity": "medium", "tags": [ "Elastic", + "Cloud", "GCP", "Continuous Monitoring", "SecOps", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_storage_bucket_permissions_modified.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_storage_bucket_permissions_modified.json index ad6beb9383eeaf..0687bb1e5178a2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_storage_bucket_permissions_modified.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_gcp_storage_bucket_permissions_modified.json @@ -22,6 +22,7 @@ "severity": "medium", "tags": [ "Elastic", + "Cloud", "GCP", "Continuous Monitoring", "SecOps", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_guardduty_detector_deletion.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_guardduty_detector_deletion.json index 3910b8e4039ada..50ee5a902b1444 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_guardduty_detector_deletion.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_guardduty_detector_deletion.json @@ -25,11 +25,12 @@ "rule_id": "523116c0-d89d-4d7c-82c2-39e6845a78ef", "severity": "high", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Monitoring", - "Continuous Monitoring" + "Monitoring" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_hex_encoding_or_decoding_activity.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_hex_encoding_or_decoding_activity.json index f5345b2276e8a7..6d3d6f456da4cc 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_hex_encoding_or_decoding_activity.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_hex_encoding_or_decoding_activity.json @@ -20,7 +20,10 @@ "severity": "low", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection", + "Defense Evasion" ], "threat": [ { @@ -55,5 +58,5 @@ } ], "type": "query", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_hidden_file_dir_tmp.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_hidden_file_dir_tmp.json index e66968a50709e8..c21c15909d82a0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_hidden_file_dir_tmp.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_hidden_file_dir_tmp.json @@ -21,7 +21,10 @@ "severity": "medium", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection", + "Defense Evasion" ], "threat": [ { @@ -56,5 +59,5 @@ } ], "type": "query", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_iis_httplogging_disabled.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_iis_httplogging_disabled.json index 507260f04d016b..7d75f508561253 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_iis_httplogging_disabled.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_iis_httplogging_disabled.json @@ -12,13 +12,16 @@ "license": "Elastic License", "max_signals": 33, "name": "IIS HTTP Logging Disabled", - "query": "event.category:process and event.type:(start or process_started) and (process.name:appcmd.exe or process.pe.original_file_name:appcmd.exe) and process.args:/dontLog\\:\\\"True\\\" and not process.parent.name:iissetup.exe", + "query": "event.category:process and event.type:(start or process_started) and (process.name:appcmd.exe or process.pe.original_file_name:appcmd.exe or winlog.event_data.OriginalFileName:appcmd.exe) and process.args:/dontLog\\:\\\"True\\\" and not process.parent.name:iissetup.exe", "risk_score": 73, "rule_id": "ebf1adea-ccf2-4943-8b96-7ab11ca173a5", "severity": "high", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_injection_msbuild.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_injection_msbuild.json index 9abce01769e921..7978ef90042713 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_injection_msbuild.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_injection_msbuild.json @@ -18,7 +18,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { @@ -53,5 +56,5 @@ } ], "type": "query", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/c2_installutil_beacon.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_installutil_beacon.json similarity index 67% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/c2_installutil_beacon.json rename to x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_installutil_beacon.json index 7437bf27141ec9..231ed1b36dc0f2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/c2_installutil_beacon.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_installutil_beacon.json @@ -11,13 +11,16 @@ "language": "eql", "license": "Elastic License", "name": "InstallUtil Process Making Network Connections", - "query": "/* this can be done without a sequence however, this does include more info on the process */\n\nsequence by process.entity_id\n [process where event.type in (\"start\", \"process_started\") and process.name == \"installutil.exe\"]\n [network where event.type == \"connection\" and process.name == \"installutil.exe\" and network.direction == \"outgoing\"]\n", + "query": "/* the benefit of doing this as an eql sequence vs kql is this will limit to alerting only on the first network connection */\n\nsequence by process.entity_id\n [process where event.type in (\"start\", \"process_started\") and process.name : \"installutil.exe\"]\n [network where event.type == \"connection\" and process.name : \"installutil.exe\" and network.direction == \"outgoing\"]\n", "risk_score": 21, "rule_id": "a13167f1-eec2-4015-9631-1fee60406dcf", "severity": "medium", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_kernel_module_removal.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_kernel_module_removal.json index ad751a10314378..5c38974b465253 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_kernel_module_removal.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_kernel_module_removal.json @@ -23,7 +23,10 @@ "severity": "high", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection", + "Defense Evasion" ], "threat": [ { @@ -58,5 +61,5 @@ } ], "type": "query", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_masquerading_as_elastic_endpoint_process.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_masquerading_as_elastic_endpoint_process.json index 8b7ef47443e2fd..163c7e834ba342 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_masquerading_as_elastic_endpoint_process.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_masquerading_as_elastic_endpoint_process.json @@ -17,7 +17,10 @@ "severity": "medium", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_masquerading_renamed_autoit.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_masquerading_renamed_autoit.json index cc964bfdd3e921..be83f8c41a2ea3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_masquerading_renamed_autoit.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_masquerading_renamed_autoit.json @@ -11,13 +11,16 @@ "language": "lucene", "license": "Elastic License", "name": "Renamed AutoIt Scripts Interpreter", - "query": "event.category:process AND event.type:(start OR process_started) AND process.pe.original_file_name:/[aA][uU][tT][oO][iI][tT]\\d\\.[eE][xX][eE]/ AND NOT process.name:/[aA][uU][tT][oO][iI][tT]\\d{1,3}\\.[eE][xX][eE]/", + "query": "event.category:process AND event.type:(start OR process_started) AND (process.pe.original_file_name:/[aA][uU][tT][oO][iI][tT]\\d\\.[eE][xX][eE]/ OR winlog.event_data.OriginalFileName:/[aA][uU][tT][oO][iI][tT]\\d\\.[eE][xX][eE]/) AND NOT process.name:/[aA][uU][tT][oO][iI][tT]\\d{1,3}\\.[eE][xX][eE]/", "risk_score": 47, "rule_id": "2e1e835d-01e5-48ca-b9fc-7a61f7f11902", "severity": "medium", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_masquerading_suspicious_werfault_childproc.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_masquerading_suspicious_werfault_childproc.json index 3000e7ac86daae..df0ecf1d534a1c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_masquerading_suspicious_werfault_childproc.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_masquerading_suspicious_werfault_childproc.json @@ -24,7 +24,10 @@ "severity": "medium", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_masquerading_werfault.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_masquerading_werfault.json index db421146085ffd..9f5615d4663745 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_masquerading_werfault.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_masquerading_werfault.json @@ -24,7 +24,10 @@ "severity": "medium", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_misc_lolbin_connecting_to_the_internet.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_misc_lolbin_connecting_to_the_internet.json index 5b5f69a0aef744..3d87720818ff50 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_misc_lolbin_connecting_to_the_internet.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_misc_lolbin_connecting_to_the_internet.json @@ -17,7 +17,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { @@ -52,5 +55,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_modification_of_boot_config.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_modification_of_boot_config.json index 6025fc5ca6452d..84ccc522496226 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_modification_of_boot_config.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_modification_of_boot_config.json @@ -17,7 +17,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { @@ -37,5 +40,5 @@ } ], "type": "query", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/c2_msbuild_beacon_sequence.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_msbuild_beacon_sequence.json similarity index 79% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/c2_msbuild_beacon_sequence.json rename to x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_msbuild_beacon_sequence.json index 59295c3735a3a4..0740e26d3bba67 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/c2_msbuild_beacon_sequence.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_msbuild_beacon_sequence.json @@ -11,13 +11,16 @@ "language": "eql", "license": "Elastic License", "name": "MsBuild Network Connection Sequence", - "query": "sequence by process.entity_id\n [process where event.type in (\"start\", \"process_started\") and process.name == \"MSBuild.exe\"]\n [network where process.name == \"MSBuild.exe\" and\n not (destination.address == \"127.0.0.1\" and source.address == \"127.0.0.1\")]\n", + "query": "sequence by process.entity_id\n [process where event.type in (\"start\", \"process_started\") and process.name : \"MSBuild.exe\"]\n [network where process.name : \"MSBuild.exe\" and\n not (destination.ip == \"127.0.0.1\" and source.ip == \"127.0.0.1\")]\n", "risk_score": 21, "rule_id": "9dc6ed5d-62a9-4feb-a903-fafa1d33b8e9", "severity": "medium", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/c2_mshta_beacon.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_mshta_beacon.json similarity index 69% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/c2_mshta_beacon.json rename to x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_mshta_beacon.json index 105f5366287772..fd19942a33d48e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/c2_mshta_beacon.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_mshta_beacon.json @@ -11,13 +11,16 @@ "language": "eql", "license": "Elastic License", "name": "Mshta Making Network Connections", - "query": "sequence by process.entity_id with maxspan=2h\n [process where event.type in (\"start\", \"process_started\") and process.name == \"mshta.exe\" and\n process.parent.name != \"Microsoft.ConfigurationManagement.exe\" and\n process.parent.executable not in (\"C:\\\\Amazon\\\\Amazon Assistant\\\\amazonAssistantService.exe\",\n \"C:\\\\TeamViewer\\\\TeamViewer.exe\") and\n process.args != \"ADSelfService_Enroll.hta\"]\n [network where process.name == \"mshta.exe\"]\n", + "query": "sequence by process.entity_id with maxspan=2h\n [process where event.type in (\"start\", \"process_started\") and process.name : \"mshta.exe\" and\n not process.parent.name : \"Microsoft.ConfigurationManagement.exe\" and\n not (process.parent.executable : \"C:\\\\Amazon\\\\Amazon Assistant\\\\amazonAssistantService.exe\" or\n process.parent.executable : \"C:\\\\TeamViewer\\\\TeamViewer.exe\") and\n not process.args : \"ADSelfService_Enroll.hta\"]\n [network where process.name : \"mshta.exe\"]\n", "risk_score": 21, "rule_id": "c2d90150-0133-451c-a783-533e736c12d7", "severity": "medium", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/c2_msxsl_beacon.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_msxsl_beacon.json similarity index 79% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/c2_msxsl_beacon.json rename to x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_msxsl_beacon.json index 27704b3e182ed6..c1883873042140 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/c2_msxsl_beacon.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_msxsl_beacon.json @@ -11,13 +11,16 @@ "language": "eql", "license": "Elastic License", "name": "MsXsl Making Network Connections", - "query": "sequence by process.entity_id\n [process where event.type in (\"start\", \"process_started\") and process.name == \"msxsl.exe\"]\n [network where event.type == \"connection\" and process.name == \"msxsl.exe\" and network.direction == \"outgoing\"]\n", + "query": "sequence by process.entity_id\n [process where event.type in (\"start\", \"process_started\") and process.name : \"msxsl.exe\"]\n [network where event.type == \"connection\" and process.name : \"msxsl.exe\" and network.direction == \"outgoing\"]\n", "risk_score": 21, "rule_id": "870d1753-1078-403e-92d4-735f142edcca", "severity": "medium", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_network_connection_from_windows_binary.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_network_connection_from_windows_binary.json new file mode 100644 index 00000000000000..4e1d0cad0b5da7 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_network_connection_from_windows_binary.json @@ -0,0 +1,44 @@ +{ + "author": [ + "Elastic" + ], + "description": "Identifies network activity from unexpected system applications. This may indicate adversarial activity as these applications are often leveraged by adversaries to execute code and evade detection.", + "from": "now-9m", + "index": [ + "logs-endpoint.events.*", + "winlogbeat-*" + ], + "language": "eql", + "license": "Elastic License", + "name": "Unusual Network Activity from a Windows System Binary", + "query": "sequence by process.entity_id with maxspan=5m\n [process where event.type in (\"start\", \"process_started\") and\n\n /* known applocker bypasses */\n (process.name : \"bginfo.exe\" or\n process.name : \"cdb.exe\" or\n process.name : \"control.exe\" or\n process.name : \"cmstp.exe\" or\n process.name : \"csi.exe\" or\n process.name : \"dnx.exe\" or\n process.name : \"fsi.exe\" or\n process.name : \"ieexec.exe\" or\n process.name : \"iexpress.exe\" or\n process.name : \"installutil.exe\" or\n process.name : \"Microsoft.Workflow.Compiler.exe\" or\n process.name : \"MSBuild.exe\" or\n process.name : \"msdt.exe\" or\n process.name : \"mshta.exe\" or\n process.name : \"msiexec.exe\" or\n process.name : \"msxsl.exe\" or\n process.name : \"odbcconf.exe\" or\n process.name : \"rcsi.exe\" or\n process.name : \"regsvr32.exe\" or\n process.name : \"xwizard.exe\")]\n [network where event.type == \"connection\" and\n (process.name : \"bginfo.exe\" or\n process.name : \"cdb.exe\" or\n process.name : \"control.exe\" or\n process.name : \"cmstp.exe\" or\n process.name : \"csi.exe\" or\n process.name : \"dnx.exe\" or\n process.name : \"fsi.exe\" or\n process.name : \"ieexec.exe\" or\n process.name : \"iexpress.exe\" or\n process.name : \"installutil.exe\" or\n process.name : \"Microsoft.Workflow.Compiler.exe\" or\n process.name : \"MSBuild.exe\" or\n process.name : \"msdt.exe\" or\n process.name : \"mshta.exe\" or\n process.name : \"msiexec.exe\" or\n process.name : \"msxsl.exe\" or\n process.name : \"odbcconf.exe\" or\n process.name : \"rcsi.exe\" or\n process.name : \"regsvr32.exe\" or\n process.name : \"xwizard.exe\")]\n", + "risk_score": 21, + "rule_id": "1fe3b299-fbb5-4657-a937-1d746f2c711a", + "severity": "medium", + "tags": [ + "Elastic", + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" + ], + "threat": [ + { + "framework": "MITRE ATT&CK", + "tactic": { + "id": "TA0005", + "name": "Defense Evasion", + "reference": "https://attack.mitre.org/tactics/TA0005/" + }, + "technique": [ + { + "id": "T1127", + "name": "Trusted Developer Utilities Proxy Execution", + "reference": "https://attack.mitre.org/techniques/T1127/" + } + ] + } + ], + "type": "eql", + "version": 1 +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_network_watcher_deletion.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_network_watcher_deletion.json index 09bbba5a049e43..0e6d9172eb2c1e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_network_watcher_deletion.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_network_watcher_deletion.json @@ -23,10 +23,11 @@ "severity": "medium", "tags": [ "Elastic", + "Cloud", "Azure", - "SecOps", "Continuous Monitoring", - "Network" + "SecOps", + "Network Security" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/c2_reg_beacon.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_reg_beacon.json similarity index 66% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/c2_reg_beacon.json rename to x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_reg_beacon.json index 332c719eaa41da..aa4f9985f6e2cc 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/c2_reg_beacon.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_reg_beacon.json @@ -11,13 +11,16 @@ "language": "eql", "license": "Elastic License", "name": "Registration Tool Making Network Connections", - "query": "sequence by process.entity_id\n [process where event.type in (\"start\", \"process_started\") and\n process.name in (\"regasm.exe\", \"regsvcs.exe\", \"regsvr32.exe\")]\n [network where event.type == \"connection\" and process.name in (\"regasm.exe\", \"regsvcs.exe\", \"regsvr32.exe\")]\nuntil\n [process where event.type == \"end\" and process.name in (\"regasm.exe\", \"regsvcs.exe\", \"regsvr32.exe\")]\n", + "query": "sequence by process.entity_id\n [process where event.type in (\"start\", \"process_started\") and\n (process.name : \"RegAsm.exe\" or process.name : \"regsvcs.exe\" or process.name : \"regsvr32.exe\")]\n [network where event.type == \"connection\" and\n (process.name : \"RegAsm.exe\" or process.name : \"regsvcs.exe\" or process.name : \"regsvr32.exe\")]\nuntil\n [process where event.type == \"end\" and\n (process.name : \"RegAsm.exe\" or process.name : \"regsvcs.exe\" or process.name : \"regsvr32.exe\")]\n", "risk_score": 21, "rule_id": "6d3456a5-4a42-49d1-aaf2-7b1fd475b2c6", "severity": "medium", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/evasion_rundll32_no_arguments.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_rundll32_no_arguments.json similarity index 61% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/evasion_rundll32_no_arguments.json rename to x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_rundll32_no_arguments.json index 06fad7e0f630b5..2950b792219b64 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/evasion_rundll32_no_arguments.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_rundll32_no_arguments.json @@ -11,13 +11,16 @@ "language": "eql", "license": "Elastic License", "name": "Unusual Child Processes of RunDLL32", - "query": "sequence with maxspan=1h\n [process where event.type in (\"start\", \"process_started\") and\n (process.name == \"rundll32.exe\" or process.pe.original_file_name == \"rundll32.exe\") and\n\n /* zero arguments excluding the binary itself (and accounting for when the binary may not be logged in args) */\n ((process.args == \"rundll32.exe\" and process.args_count == 1) or\n (process.args != \"rundll32.exe\" and process.args_count == 0))\n\n ] by process.entity_id\n [process where event.type in (\"start\", \"process_started\") and\n (process.name == \"rundll32.exe\" or process.pe.original_file_name == \"rundll32.exe\")\n ] by process.parent.entity_id\n", + "query": "sequence with maxspan=1h\n [process where event.type in (\"start\", \"process_started\") and\n /* uncomment once in winlogbeat */\n (process.name : \"rundll32.exe\" /* or process.pe.original_file_name == \"RUNDLL32.EXE\" */ ) and\n process.args_count < 2\n ] by process.entity_id\n [process where event.type in (\"start\", \"process_started\") and\n /* uncomment once in winlogbeat */\n (process.name : \"rundll32.exe\" /* or process.pe.original_file_name == \"RUNDLL32.EXE\" */ )\n ] by process.parent.entity_id\n", "risk_score": 21, "rule_id": "f036953a-4615-4707-a1ca-dc53bf69dcd5", "severity": "high", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/c2_rundll32_sequence.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_rundll32_sequence.json similarity index 67% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/c2_rundll32_sequence.json rename to x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_rundll32_sequence.json index 6f465325039a6b..c022d0a603858f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/c2_rundll32_sequence.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_rundll32_sequence.json @@ -11,13 +11,16 @@ "language": "eql", "license": "Elastic License", "name": "Unusual Network Connection Sequence via RunDLL32", - "query": "sequence by process.entity_id with maxspan=2h\n [process where event.type in (\"start\", \"process_started\") and\n (process.name == \"rundll32.exe\" or process.pe.original_file_name == \"rundll32.exe\") and\n\n /* zero arguments excluding the binary itself (and accounting for when the binary may not be logged in args) */\n ((process.args == \"rundll32.exe\" and process.args_count == 1) or\n (process.args != \"rundll32.exe\" and process.args_count == 0))]\n\n [network where event.type == \"connection\" and\n (process.name == \"rundll32.exe\" or process.pe.original_file_name == \"rundll32.exe\")]\n", + "query": "sequence by process.entity_id with maxspan=2h\n [process where event.type in (\"start\", \"process_started\") and\n /* uncomment once in winlogbeat */\n (process.name : \"rundll32.exe\" /* or process.pe.original_file_name == \"RUNDLL32.EXE\" */ ) and\n process.args_count < 2]\n [network where event.type == \"connection\" and\n /* uncomment once in winlogbeat */\n (process.name : \"rundll32.exe\" /* or process.pe.original_file_name == \"RUNDLL32.EXE\" */ )]\n", "risk_score": 21, "rule_id": "2b347f66-6739-4ae3-bd94-195036dde8b3", "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_s3_bucket_configuration_deletion.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_s3_bucket_configuration_deletion.json index adbe310b784e51..6cfdf407813975 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_s3_bucket_configuration_deletion.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_s3_bucket_configuration_deletion.json @@ -28,11 +28,12 @@ "rule_id": "227dc608-e558-43d9-b521-150772250bae", "severity": "low", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Asset Visibility", - "Continuous Monitoring" + "Asset Visibility" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_sdelete_like_filename_rename.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_sdelete_like_filename_rename.json index ec3030d44ff294..6fea9a3c789456 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_sdelete_like_filename_rename.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_sdelete_like_filename_rename.json @@ -18,7 +18,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_suspicious_managedcode_host_process.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_suspicious_managedcode_host_process.json index 1bb3f26c0298f3..85eac81808a71d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_suspicious_managedcode_host_process.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_suspicious_managedcode_host_process.json @@ -20,7 +20,10 @@ "severity": "high", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_suspicious_okta_user_password_reset_or_unlock_attempts.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_suspicious_okta_user_password_reset_or_unlock_attempts.json similarity index 99% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_suspicious_okta_user_password_reset_or_unlock_attempts.json rename to x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_suspicious_okta_user_password_reset_or_unlock_attempts.json index 5f115416fa0325..fedeaca68ab643 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/credential_access_suspicious_okta_user_password_reset_or_unlock_attempts.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_suspicious_okta_user_password_reset_or_unlock_attempts.json @@ -25,6 +25,7 @@ "severity": "medium", "tags": [ "Elastic", + "Identity", "Okta", "Continuous Monitoring", "SecOps", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/evasion_suspicious_scrobj_load.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_suspicious_scrobj_load.json similarity index 51% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/evasion_suspicious_scrobj_load.json rename to x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_suspicious_scrobj_load.json index 7880b86533b538..16364f590cd0e1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/evasion_suspicious_scrobj_load.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_suspicious_scrobj_load.json @@ -11,13 +11,16 @@ "language": "eql", "license": "Elastic License", "name": "Windows Suspicious Script Object Execution", - "query": "sequence by process.entity_id with maxspan=2m\n [process where event.type in (\"start\", \"process_started\") and\n /* process.code_signature.* fields need to be populated for 7.10 */\n process.code_signature.subject_name == \"Microsoft Corporation\" and process.code_signature.trusted == true and\n process.name not in (\"cscript.exe\",\n \"iexplore.exe\",\n \"MicrosoftEdge.exe\",\n \"msiexec.exe\",\n \"smartscreen.exe\",\n \"taskhostw.exe\",\n \"w3wp.exe\",\n \"wscript.exe\")]\n [library where event.type == \"start\" and file.name == \"scrobj.dll\"]\n", + "query": "/* add winlogbeat-* when process.code_signature.* fields are populated */\n\nsequence by process.entity_id with maxspan=2m\n [process where event.type in (\"start\", \"process_started\") and\n /* uncomment once in winlogbeat */\n /* process.code_signature.subject_name == \"Microsoft Corporation\" and process.code_signature.trusted == true and */\n not (process.name : \"cscript.exe\" or\n process.name : \"iexplore.exe\" or\n process.name : \"MicrosoftEdge.exe\" or\n process.name : \"msiexec.exe\" or\n process.name : \"smartscreen.exe\" or\n process.name : \"taskhostw.exe\" or\n process.name : \"w3wp.exe\" or\n process.name : \"wscript.exe\")]\n [library where event.type == \"start\" and file.name : \"scrobj.dll\"]\n", "risk_score": 21, "rule_id": "4ed678a9-3a4f-41fb-9fea-f85a6e0a0dff", "severity": "medium", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/evasion_suspicious_wmi_script.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_suspicious_wmi_script.json similarity index 59% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/evasion_suspicious_wmi_script.json rename to x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_suspicious_wmi_script.json index 943471f5801c21..e9224162643595 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/evasion_suspicious_wmi_script.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_suspicious_wmi_script.json @@ -5,19 +5,21 @@ "description": "Identifies WMIC whitelisting bypass techniques by alerting on suspicious execution of scripts. When WMIC loads scripting libraries it may be indicative of a whitelist bypass.", "from": "now-9m", "index": [ - "logs-endpoint.events.*", - "winlogbeat-*" + "logs-endpoint.events.*" ], "language": "eql", "license": "Elastic License", "name": "Suspicious WMIC XSL Script Execution", - "query": "/* lots of wildcards in the args\n need to verify args cleanup is accurate\n*/\n\nsequence by process.entity_id with maxspan=2m\n[process where event.type in (\"start\", \"process_started\") and\n (process.name == \"wmic.exe\" or process.pe.original_file_name == \"wmic.exe\") and\n wildcard(process.args, \"format*:*\", \"/format*:*\", \"*-format*:*\") and\n not process.args in (\"/format:table\", \"/format:table\") or wildcard(process.args, \"format*:*\")]\n[library where event.type == \"start\" and file.name in (\"jscript.dll\", \"vbscript.dll\")]\n", + "query": "sequence by process.entity_id with maxspan=2m\n[process where event.type in (\"start\", \"process_started\") and\n (process.name : \"WMIC.exe\" or process.pe.original_file_name == \"wmic.exe\") and\n wildcard(process.args, \"format*:*\", \"/format*:*\", \"*-format*:*\") and\n not wildcard(process.command_line, \"* /format:table *\")]\n[library where event.type == \"start\" and file.name in (\"jscript.dll\", \"vbscript.dll\")]\n", "risk_score": 21, "rule_id": "7f370d54-c0eb-4270-ac5a-9a6020585dc6", "severity": "medium", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_suspicious_zoom_child_process.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_suspicious_zoom_child_process.json index 7b08f5a565424c..f3c20e5251184e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_suspicious_zoom_child_process.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_suspicious_zoom_child_process.json @@ -20,7 +20,10 @@ "severity": "medium", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_system_critical_proc_abnormal_file_activity.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_system_critical_proc_abnormal_file_activity.json index 6fea3a75c8e625..31702332d7cd47 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_system_critical_proc_abnormal_file_activity.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_system_critical_proc_abnormal_file_activity.json @@ -17,7 +17,10 @@ "severity": "high", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_unusual_system_vp_child_program.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_unusual_system_vp_child_program.json index 4efec948f49a7f..65439d6d653b67 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_unusual_system_vp_child_program.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_unusual_system_vp_child_program.json @@ -17,7 +17,10 @@ "severity": "high", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_via_filter_manager.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_via_filter_manager.json index 210e9c778afef7..3092ea653aa4be 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_via_filter_manager.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_via_filter_manager.json @@ -4,7 +4,8 @@ ], "description": "The Filter Manager Control Program (fltMC.exe) binary may be abused by adversaries to unload a filter driver and evade defenses.", "index": [ - "winlogbeat-*" + "winlogbeat-*", + "logs-endpoint.events.*" ], "language": "kuery", "license": "Elastic License", @@ -15,7 +16,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_volume_shadow_copy_deletion_via_vssadmin.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_volume_shadow_copy_deletion_via_vssadmin.json index 8a504281b03f76..493ee919000dc2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_volume_shadow_copy_deletion_via_vssadmin.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_volume_shadow_copy_deletion_via_vssadmin.json @@ -17,7 +17,10 @@ "severity": "high", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { @@ -37,5 +40,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_volume_shadow_copy_deletion_via_wmic.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_volume_shadow_copy_deletion_via_wmic.json index 2ae938bb34104d..49a68f4bfcf661 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_volume_shadow_copy_deletion_via_wmic.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_volume_shadow_copy_deletion_via_wmic.json @@ -17,7 +17,10 @@ "severity": "high", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Defense Evasion" ], "threat": [ { @@ -37,5 +40,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_waf_acl_deletion.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_waf_acl_deletion.json index 32101029fb1072..86ed9e2f9c042d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_waf_acl_deletion.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_waf_acl_deletion.json @@ -25,11 +25,12 @@ "rule_id": "91d04cd4-47a9-4334-ab14-084abe274d49", "severity": "medium", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Network", - "Continuous Monitoring" + "Network Security" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_waf_rule_or_rule_group_deletion.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_waf_rule_or_rule_group_deletion.json index a08c05d0d6ca7a..302e89d416f4bc 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_waf_rule_or_rule_group_deletion.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_waf_rule_or_rule_group_deletion.json @@ -25,11 +25,12 @@ "rule_id": "5beaebc1-cc13-4bfc-9949-776f9e0dc318", "severity": "medium", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Network", - "Continuous Monitoring" + "Network Security" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_blob_container_access_mod.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_blob_container_access_mod.json index 7e601c9928d08b..16db02338de55e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_blob_container_access_mod.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_blob_container_access_mod.json @@ -23,9 +23,10 @@ "severity": "low", "tags": [ "Elastic", + "Cloud", "Azure", - "SecOps", "Continuous Monitoring", + "SecOps", "Asset Visibility" ], "threat": [ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_kernel_module_enumeration.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_kernel_module_enumeration.json index af9c4b5409964d..f2681f5501af0b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_kernel_module_enumeration.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_kernel_module_enumeration.json @@ -20,7 +20,10 @@ "severity": "medium", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection", + "Discovery" ], "threat": [ { @@ -40,5 +43,5 @@ } ], "type": "query", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_net_command_system_account.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_net_command_system_account.json index 96c300cfde0160..682434f28f7454 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_net_command_system_account.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_net_command_system_account.json @@ -17,7 +17,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Discovery" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_post_exploitation_public_ip_reconnaissance.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_post_exploitation_public_ip_reconnaissance.json index 952d70ee3589a3..36e95cf4ade417 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_post_exploitation_public_ip_reconnaissance.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_post_exploitation_public_ip_reconnaissance.json @@ -24,8 +24,8 @@ "tags": [ "Elastic", "Network", - "Threat Detection, Preventing and Hunting", - "Post-Execution" + "Threat Detection", + "Discovery" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_process_discovery_via_tasklist_command.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_process_discovery_via_tasklist_command.json index c2d95de4129f95..59ddc00d6525d3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_process_discovery_via_tasklist_command.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_process_discovery_via_tasklist_command.json @@ -7,7 +7,8 @@ "Administrators may use the tasklist command to display a list of currently running processes. By itself, it does not indicate malicious activity. After obtaining a foothold, it's possible adversaries may use discovery commands like tasklist to get information about running processes." ], "index": [ - "winlogbeat-*" + "winlogbeat-*", + "logs-endpoint.events.*" ], "language": "kuery", "license": "Elastic License", @@ -18,7 +19,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Discovery" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_virtual_machine_fingerprinting.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_virtual_machine_fingerprinting.json index d913a92e2ee0e3..66b744667f5685 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_virtual_machine_fingerprinting.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_virtual_machine_fingerprinting.json @@ -20,7 +20,10 @@ "severity": "high", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection", + "Discovery" ], "threat": [ { @@ -40,5 +43,5 @@ } ], "type": "query", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_whoami_command_activity.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_whoami_command_activity.json index cb330879be9b3d..2e577e44c960e2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_whoami_command_activity.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_whoami_command_activity.json @@ -7,7 +7,8 @@ "Some normal use of this program, at varying levels of frequency, may originate from scripts, automation tools and frameworks. Usage by non-engineers and ordinary users is unusual." ], "index": [ - "winlogbeat-*" + "winlogbeat-*", + "logs-endpoint.events.*" ], "language": "kuery", "license": "Elastic License", @@ -18,7 +19,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Discovery" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_whoami_commmand.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_whoami_commmand.json index a8b34362d95793..ddf309b3776093 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_whoami_commmand.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/discovery_whoami_commmand.json @@ -20,7 +20,10 @@ "severity": "low", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection", + "Discovery" ], "threat": [ { @@ -40,5 +43,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/elastic_endpoint.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/elastic_endpoint.json index f3acc5d3a2b5e2..d6e7419391f9c4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/elastic_endpoint.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/elastic_endpoint.json @@ -60,7 +60,7 @@ ], "tags": [ "Elastic", - "Endpoint" + "Endpoint Security" ], "timestamp_override": "event.ingested", "type": "query", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_adversary_behavior_detected.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_adversary_behavior_detected.json index b3bac305bc1f1f..8084067b3a6d28 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_adversary_behavior_detected.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_adversary_behavior_detected.json @@ -17,7 +17,7 @@ "severity": "medium", "tags": [ "Elastic", - "Endpoint" + "Endpoint Security" ], "type": "query", "version": 4 diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_cred_dumping_detected.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_cred_dumping_detected.json index 2f91c1fe813f97..9c28d065b322d4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_cred_dumping_detected.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_cred_dumping_detected.json @@ -17,7 +17,7 @@ "severity": "high", "tags": [ "Elastic", - "Endpoint" + "Endpoint Security" ], "type": "query", "version": 4 diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_cred_dumping_prevented.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_cred_dumping_prevented.json index 75488c2d3a5ed7..352712e38f42de 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_cred_dumping_prevented.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_cred_dumping_prevented.json @@ -17,7 +17,7 @@ "severity": "medium", "tags": [ "Elastic", - "Endpoint" + "Endpoint Security" ], "type": "query", "version": 4 diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_cred_manipulation_detected.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_cred_manipulation_detected.json index adc29d9106774f..259bcd51aeb3e2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_cred_manipulation_detected.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_cred_manipulation_detected.json @@ -17,7 +17,7 @@ "severity": "high", "tags": [ "Elastic", - "Endpoint" + "Endpoint Security" ], "type": "query", "version": 4 diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_cred_manipulation_prevented.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_cred_manipulation_prevented.json index 99def69978a488..19348062b10f1d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_cred_manipulation_prevented.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_cred_manipulation_prevented.json @@ -17,7 +17,7 @@ "severity": "medium", "tags": [ "Elastic", - "Endpoint" + "Endpoint Security" ], "type": "query", "version": 4 diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_exploit_detected.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_exploit_detected.json index 80eb3ce637f30e..2fd3aaa0d8a576 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_exploit_detected.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_exploit_detected.json @@ -17,7 +17,7 @@ "severity": "high", "tags": [ "Elastic", - "Endpoint" + "Endpoint Security" ], "type": "query", "version": 4 diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_exploit_prevented.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_exploit_prevented.json index 50444904654de8..8f90e1162546b0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_exploit_prevented.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_exploit_prevented.json @@ -17,7 +17,7 @@ "severity": "medium", "tags": [ "Elastic", - "Endpoint" + "Endpoint Security" ], "type": "query", "version": 4 diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_malware_detected.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_malware_detected.json index bb2ddf92a83e7b..3d740f8b7064f4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_malware_detected.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_malware_detected.json @@ -17,7 +17,7 @@ "severity": "critical", "tags": [ "Elastic", - "Endpoint" + "Endpoint Security" ], "type": "query", "version": 4 diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_malware_prevented.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_malware_prevented.json index fae8a3a0ab5a67..33195c7fcbecca 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_malware_prevented.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_malware_prevented.json @@ -17,7 +17,7 @@ "severity": "high", "tags": [ "Elastic", - "Endpoint" + "Endpoint Security" ], "type": "query", "version": 4 diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_permission_theft_detected.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_permission_theft_detected.json index 821c3b0d8a63bb..fac13a6d358dd2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_permission_theft_detected.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_permission_theft_detected.json @@ -17,7 +17,7 @@ "severity": "high", "tags": [ "Elastic", - "Endpoint" + "Endpoint Security" ], "type": "query", "version": 4 diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_permission_theft_prevented.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_permission_theft_prevented.json index e38afe19e7d385..a2d8700076c234 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_permission_theft_prevented.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_permission_theft_prevented.json @@ -17,7 +17,7 @@ "severity": "medium", "tags": [ "Elastic", - "Endpoint" + "Endpoint Security" ], "type": "query", "version": 4 diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_process_injection_detected.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_process_injection_detected.json index 52eb3c2d96bf71..ef4f29067b0c5f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_process_injection_detected.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_process_injection_detected.json @@ -17,7 +17,7 @@ "severity": "high", "tags": [ "Elastic", - "Endpoint" + "Endpoint Security" ], "type": "query", "version": 4 diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_process_injection_prevented.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_process_injection_prevented.json index 76aff15e1588c1..b22751e35c053d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_process_injection_prevented.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_process_injection_prevented.json @@ -17,7 +17,7 @@ "severity": "medium", "tags": [ "Elastic", - "Endpoint" + "Endpoint Security" ], "type": "query", "version": 4 diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_ransomware_detected.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_ransomware_detected.json index 29efdd910904d3..3b973f42bbca52 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_ransomware_detected.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_ransomware_detected.json @@ -17,7 +17,7 @@ "severity": "critical", "tags": [ "Elastic", - "Endpoint" + "Endpoint Security" ], "type": "query", "version": 4 diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_ransomware_prevented.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_ransomware_prevented.json index c603e503c5dad4..b6458b73e80153 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_ransomware_prevented.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/endpoint_ransomware_prevented.json @@ -17,7 +17,7 @@ "severity": "high", "tags": [ "Elastic", - "Endpoint" + "Endpoint Security" ], "type": "query", "version": 4 diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/escalation_uac_sdclt.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/escalation_uac_sdclt.json deleted file mode 100644 index 843ba3401b4e40..00000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/escalation_uac_sdclt.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "author": [ - "Elastic" - ], - "description": "Identifies User Account Control (UAC) bypass via sdclt.exe. Attackers bypass UAC to stealthily execute code with elevated permissions.", - "from": "now-9m", - "index": [ - "logs-endpoint.events.*", - "winlogbeat-*" - ], - "language": "eql", - "license": "Elastic License", - "name": "Bypass UAC via Sdclt", - "query": "sequence with maxspan=1m\n [process where event.type in (\"start\", \"process_started\") and process.name == \"sdclt.exe\" and\n /* process.code_signature.* fields need to be populated for 7.10 */\n process.code_signature.subject_name == \"Microsoft Corporation\" and process.code_signature.trusted == true and\n process.args == \"/kickoffelev\"\n ] by process.entity_id\n [process where event.type in (\"start\", \"process_started\") and process.parent.name == \"sdclt.exe\" and\n process.executable not in (\"C:\\\\Windows\\\\System32\\\\sdclt.exe\",\n \"C:\\\\Windows\\\\System32\\\\control.exe\",\n \"C:\\\\Windows\\\\SysWOW64\\\\sdclt.exe\",\n \"C:\\\\Windows\\\\SysWOW64\\\\control.exe\")\n ] by process.parent.entity_id\n", - "risk_score": 21, - "rule_id": "9b54e002-034a-47ac-9307-ad12c03fa900", - "severity": "high", - "tags": [ - "Elastic", - "Windows" - ], - "threat": [ - { - "framework": "MITRE ATT&CK", - "tactic": { - "id": "TA0004", - "name": "Privilege Escalation", - "reference": "https://attack.mitre.org/tactics/TA0004/" - }, - "technique": [ - { - "id": "T1088", - "name": "Bypass User Account Control", - "reference": "https://attack.mitre.org/techniques/T1088/" - } - ] - } - ], - "type": "eql", - "version": 1 -} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_command_prompt_connecting_to_the_internet.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_command_prompt_connecting_to_the_internet.json index e9989fe50019e0..d73b1a4cab008f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_command_prompt_connecting_to_the_internet.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_command_prompt_connecting_to_the_internet.json @@ -20,7 +20,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Execution" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_command_shell_started_by_powershell.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_command_shell_started_by_powershell.json index bb252b14168320..220a7f94dce9a1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_command_shell_started_by_powershell.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_command_shell_started_by_powershell.json @@ -17,7 +17,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Execution" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_command_shell_started_by_svchost.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_command_shell_started_by_svchost.json index aeae5518fece1b..d6b1e67b119825 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_command_shell_started_by_svchost.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_command_shell_started_by_svchost.json @@ -17,7 +17,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Execution" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_command_shell_started_by_unusual_process.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_command_shell_started_by_unusual_process.json index 577bee1ffe6de6..566a5963c122a6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_command_shell_started_by_unusual_process.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_command_shell_started_by_unusual_process.json @@ -17,7 +17,10 @@ "severity": "medium", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Execution" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_command_virtual_machine.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_command_virtual_machine.json index 5e7852e1c1b132..31c4d488c6960f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_command_virtual_machine.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_command_virtual_machine.json @@ -24,11 +24,12 @@ "rule_id": "60884af6-f553-4a6c-af13-300047455491", "severity": "medium", "tags": [ - "Azure", "Elastic", - "SecOps", + "Cloud", + "Azure", "Continuous Monitoring", - "Logging" + "SecOps", + "Log Auditing" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_html_help_executable_program_connecting_to_the_internet.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_html_help_executable_program_connecting_to_the_internet.json index 963c6b2e53ed64..d33f2287c7d8b2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_html_help_executable_program_connecting_to_the_internet.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_html_help_executable_program_connecting_to_the_internet.json @@ -17,7 +17,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Execution" ], "threat": [ { @@ -52,5 +55,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_local_service_commands.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_local_service_commands.json index 7b20cefdc67f0a..693ca83e387b3e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_local_service_commands.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_local_service_commands.json @@ -17,7 +17,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Execution" ], "threat": [ { @@ -37,5 +40,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_ms_office_written_file.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_ms_office_written_file.json index 35d7a7c969ee70..2db46080a4e75f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_ms_office_written_file.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_ms_office_written_file.json @@ -11,13 +11,16 @@ "language": "eql", "license": "Elastic License", "name": "Execution of File Written or Modified by Microsoft Office", - "query": "sequence with maxspan=2h\n [file where event.type != \"delete\" and file.extension == \"exe\" and\n process.name in (\"winword.exe\",\n \"excel.exe\",\n \"outlook.exe\",\n \"powerpnt.exe\",\n \"eqnedt32.exe\",\n \"fltldr.exe\",\n \"mspub.exe\",\n \"msaccess.exe\")\n ] by host.id, file.path\n [process where event.type in (\"start\", \"process_started\")] by host.id, process.executable\n", + "query": "sequence with maxspan=2h\n [file where event.type != \"deletion\" and file.extension : \"exe\" and\n (process.name : \"WINWORD.EXE\" or\n process.name : \"EXCEL.EXE\" or\n process.name : \"OUTLOOK.EXE\" or\n process.name : \"POWERPNT.EXE\" or\n process.name : \"eqnedt32.exe\" or\n process.name : \"fltldr.exe\" or\n process.name : \"MSPUB.EXE\" or\n process.name : \"MSACCESS.EXE\")\n ] by host.id, file.path\n [process where event.type in (\"start\", \"process_started\")] by host.id, process.executable\n", "risk_score": 21, "rule_id": "0d8ad79f-9025-45d8-80c1-4f0cd3c5e8e5", "severity": "high", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Execution" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_msbuild_making_network_connections.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_msbuild_making_network_connections.json index fcbbfbdb3d686a..7fd2933fe46f14 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_msbuild_making_network_connections.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_msbuild_making_network_connections.json @@ -17,7 +17,10 @@ "severity": "medium", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Execution" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_mshta_making_network_connections.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_mshta_making_network_connections.json index 7af823070889f4..9b863c2ed5ee4d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_mshta_making_network_connections.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_mshta_making_network_connections.json @@ -20,7 +20,10 @@ "severity": "medium", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Execution" ], "threat": [ { @@ -40,5 +43,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_msxsl_network.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_msxsl_network.json index 1dc75575636fbf..17987218af0ae8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_msxsl_network.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_msxsl_network.json @@ -17,7 +17,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Execution" ], "threat": [ { @@ -37,5 +40,5 @@ } ], "type": "query", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_pdf_written_file.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_pdf_written_file.json index 3963b3d594902c..a807052cf7b0dd 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_pdf_written_file.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_pdf_written_file.json @@ -11,13 +11,16 @@ "language": "eql", "license": "Elastic License", "name": "Execution of File Written or Modified by PDF Reader", - "query": "sequence with maxspan=2h\n [file where event.type != \"delete\" and file.extension == \"exe\" and\n process.name in (\"acrord32.exe\", \"rdrcef.exe\", \"foxitphantomPDF.exe\", \"foxitreader.exe\") and\n file.name not in (\"foxitphantomPDF.exe\",\n \"FoxitPhantomPDFUpdater.exe\",\n \"foxitreader.exe\",\n \"FoxitReaderUpdater.exe\",\n \"acrord32.exe\",\n \"rdrcef.exe\")\n ] by host.id, file.path\n [process where event.type in (\"start\", \"process_started\")] by host.id, process.executable\n", + "query": "sequence with maxspan=2h\n [file where event.type != \"deletion\" and file.extension : \"exe\" and\n (process.name : \"AcroRd32.exe\" or\n process.name : \"rdrcef.exe\" or\n process.name : \"FoxitPhantomPDF.exe\" or\n process.name : \"FoxitReader.exe\") and\n not (file.name : \"FoxitPhantomPDF.exe\" or\n file.name : \"FoxitPhantomPDFUpdater.exe\" or\n file.name : \"FoxitReader.exe\" or\n file.name : \"FoxitReaderUpdater.exe\" or\n file.name : \"AcroRd32.exe\" or\n file.name : \"rdrcef.exe\")\n ] by host.id, file.path\n [process where event.type in (\"start\", \"process_started\")] by host.id, process.executable\n", "risk_score": 21, "rule_id": "1defdd62-cd8d-426e-a246-81a37751bb2b", "severity": "high", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Execution" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_perl_tty_shell.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_perl_tty_shell.json index 4502f42bbb4c4d..ee6cee941bef2e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_perl_tty_shell.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_perl_tty_shell.json @@ -17,7 +17,10 @@ "severity": "high", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection", + "Execution" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_psexec_lateral_movement_command.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_psexec_lateral_movement_command.json index f647d8d00e0844..8505a837ad591f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_psexec_lateral_movement_command.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_psexec_lateral_movement_command.json @@ -20,7 +20,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Execution" ], "threat": [ { @@ -55,5 +58,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_python_tty_shell.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_python_tty_shell.json index 0e8b5f0218d002..151c3fbb8df46d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_python_tty_shell.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_python_tty_shell.json @@ -17,7 +17,10 @@ "severity": "high", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection", + "Execution" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_register_server_program_connecting_to_the_internet.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_register_server_program_connecting_to_the_internet.json index 899bb1c20e711e..bf1a30f11137ee 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_register_server_program_connecting_to_the_internet.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_register_server_program_connecting_to_the_internet.json @@ -20,7 +20,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Execution" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_script_executing_powershell.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_script_executing_powershell.json index 6d7f11f01fae02..843cf322e58499 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_script_executing_powershell.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_script_executing_powershell.json @@ -17,7 +17,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Execution" ], "threat": [ { @@ -37,5 +40,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_suspicious_ms_office_child_process.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_suspicious_ms_office_child_process.json index 005a0c38c8a8b9..90c60ceea37abf 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_suspicious_ms_office_child_process.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_suspicious_ms_office_child_process.json @@ -17,7 +17,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Execution" ], "threat": [ { @@ -37,5 +40,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_suspicious_ms_outlook_child_process.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_suspicious_ms_outlook_child_process.json index 74e21c7d174792..d5ee8fa8183674 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_suspicious_ms_outlook_child_process.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_suspicious_ms_outlook_child_process.json @@ -17,7 +17,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Execution" ], "threat": [ { @@ -37,5 +40,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_suspicious_pdf_reader.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_suspicious_pdf_reader.json index adf1a76bfb9011..bf37f5a803e583 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_suspicious_pdf_reader.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_suspicious_pdf_reader.json @@ -17,7 +17,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Execution" ], "threat": [ { @@ -37,5 +40,5 @@ } ], "type": "query", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_suspicious_psexesvc.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_suspicious_psexesvc.json similarity index 76% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_suspicious_psexesvc.json rename to x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_suspicious_psexesvc.json index 51396fb5995f67..205b5148f2fb4d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_execution_suspicious_psexesvc.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_suspicious_psexesvc.json @@ -11,13 +11,16 @@ "language": "kuery", "license": "Elastic License", "name": "Suspicious Process Execution via Renamed PsExec Executable", - "query": "event.category:process and event.type:(start or process_started) and process.pe.original_file_name:(psexesvc.exe or PSEXESVC.exe) and process.parent.name:services.exe and not process.name:(psexesvc.exe or PSEXESVC.exe)", + "query": "event.category:process and event.type:(start or process_started) and (process.pe.original_file_name:(psexesvc.exe or PSEXESVC.exe) or winlog.event_data.OriginalFileName:(psexesvc.exe or PSEXESVC.exe)) and process.parent.name:services.exe and not process.name:(psexesvc.exe or PSEXESVC.exe)", "risk_score": 47, "rule_id": "e2f9fdf5-8076-45ad-9427-41e0e03dc9c2", "severity": "medium", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Execution" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_unusual_dns_service_children.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_unusual_dns_service_children.json index 96305b2197bfc0..52e67b0c7bcffc 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_unusual_dns_service_children.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_unusual_dns_service_children.json @@ -7,7 +7,8 @@ "Werfault.exe will legitimately spawn when dns.exe crashes, but the DNS service is very stable and so this is a low occurring event. Denial of Service (DoS) attempts by intentionally crashing the service will also cause werfault.exe to spawn." ], "index": [ - "winlogbeat-*" + "winlogbeat-*", + "logs-endpoint.events.*" ], "language": "kuery", "license": "Elastic License", @@ -24,7 +25,10 @@ "severity": "high", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Execution" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_unusual_dns_service_file_writes.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_unusual_dns_service_file_writes.json index c175ecbfa78b61..229fc28beee9c5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_unusual_dns_service_file_writes.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_unusual_dns_service_file_writes.json @@ -4,7 +4,8 @@ ], "description": "Identifies an unexpected file being modified by dns.exe, the process responsible for Windows DNS Server services, which may indicate activity related to remote code execution or other forms of exploitation.", "index": [ - "winlogbeat-*" + "winlogbeat-*", + "logs-endpoint.events.*" ], "language": "kuery", "license": "Elastic License", @@ -20,7 +21,10 @@ "severity": "high", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Execution" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_unusual_network_connection_via_rundll32.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_unusual_network_connection_via_rundll32.json index 11041593506557..2763f69e1f8e87 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_unusual_network_connection_via_rundll32.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_unusual_network_connection_via_rundll32.json @@ -17,7 +17,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Execution" ], "threat": [ { @@ -37,5 +40,5 @@ } ], "type": "query", - "version": 5 + "version": 6 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_unusual_process_network_connection.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_unusual_process_network_connection.json index 774e8e9189ced0..877c489b0d1870 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_unusual_process_network_connection.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_unusual_process_network_connection.json @@ -17,7 +17,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Execution" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_via_compiled_html_file.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_via_compiled_html_file.json index fe3e1108304206..276e5c18335f53 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_via_compiled_html_file.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_via_compiled_html_file.json @@ -7,7 +7,8 @@ "The HTML Help executable program (hh.exe) runs whenever a user clicks a compiled help (.chm) file or menu item that opens the help file inside the Help Viewer. This is not always malicious, but adversaries may abuse this technology to conceal malicious code." ], "index": [ - "winlogbeat-*" + "winlogbeat-*", + "logs-endpoint.events.*" ], "language": "kuery", "license": "Elastic License", @@ -18,7 +19,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Execution" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_via_hidden_shell_conhost.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_via_hidden_shell_conhost.json index 7fbf962469f71f..b6f6502e603aa4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_via_hidden_shell_conhost.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_via_hidden_shell_conhost.json @@ -20,7 +20,10 @@ "severity": "high", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Execution" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_via_net_com_assemblies.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_via_net_com_assemblies.json index d9dcbfe25a4c2d..0a21599c31a4ad 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_via_net_com_assemblies.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_via_net_com_assemblies.json @@ -17,7 +17,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Execution" ], "threat": [ { @@ -52,5 +55,5 @@ } ], "type": "query", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_via_system_manager.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_via_system_manager.json index 081ebcb5189991..13493a90e3e505 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_via_system_manager.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_via_system_manager.json @@ -24,11 +24,12 @@ "rule_id": "37b211e8-4e2f-440f-86d8-06cc8f158cfa", "severity": "low", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Logging", - "Continuous Monitoring" + "Log Auditing" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_via_xp_cmdshell_mssql_stored_procedure.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_via_xp_cmdshell_mssql_stored_procedure.json index 8769e641fad90f..91097b2050d172 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_via_xp_cmdshell_mssql_stored_procedure.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_via_xp_cmdshell_mssql_stored_procedure.json @@ -17,7 +17,10 @@ "severity": "high", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Execution" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_wpad_exploitation.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_wpad_exploitation.json index 03c4482b60340e..a09bce5119ecf4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_wpad_exploitation.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/execution_wpad_exploitation.json @@ -5,19 +5,21 @@ "description": "Identifies probable exploitation of the Web Proxy Auto-Discovery Protocol (WPAD) service. Attackers who have access to the local network or upstream DNS traffic can inject malicious JavaScript to the WPAD service which can lead to a full system compromise.", "from": "now-9m", "index": [ - "logs-endpoint.events.*", - "winlogbeat-*" + "logs-endpoint.events.*" ], "language": "eql", "license": "Elastic License", "name": "WPAD Service Exploit", - "query": "/* preference would be to use user.sid rather than domain+name, once it is available in ECS + datasources */\n\nsequence with maxspan=5s\n [process where event.type in (\"start\", \"process_started\") and process.name == \"svchost.exe\" and\n user.domain == \"NT AUTHORITY\" and user.name == \"LOCAL SERVICE\"] by process.entity_id\n [network where network.protocol == \"dns\" and process.name == \"svchost.exe\" and\n dns.question.name == \"wpad\" and process.name == \"svchost.exe\"] by process.entity_id\n [network where event.type == \"connection\" and process.name == \"svchost.exe\"\n and network.direction == \"outgoing\" and destination.port == 80] by process.entity_id\n [library where event.type == \"start\" and process.name == \"svchost.exe\" and\n file.name == \"jscript.dll\" and process.name == \"svchost.exe\"] by process.entity_id\n [process where event.type in (\"start\", \"process_started\") and\n process.parent.name == \"svchost.exe\"] by process.parent.entity_id\n", + "query": "/* preference would be to use user.sid rather than domain+name, once it is available in ECS + datasources */\n\nsequence with maxspan=5s\n [process where event.type in (\"start\", \"process_started\") and process.name : \"svchost.exe\" and\n user.domain == \"NT AUTHORITY\" and user.name == \"LOCAL SERVICE\"] by process.entity_id\n [network where network.protocol == \"dns\" and process.name : \"svchost.exe\" and\n dns.question.name : \"wpad\" and process.name : \"svchost.exe\"] by process.entity_id\n [network where event.type == \"connection\" and process.name : \"svchost.exe\"\n and network.direction == \"outgoing\" and destination.port == 80] by process.entity_id\n [library where event.type == \"start\" and process.name : \"svchost.exe\" and\n file.name : \"jscript.dll\" and process.name : \"svchost.exe\"] by process.entity_id\n [process where event.type in (\"start\", \"process_started\") and\n process.parent.name : \"svchost.exe\"] by process.parent.entity_id\n", "risk_score": 21, "rule_id": "ec328da1-d5df-482b-866c-4a435692b1f3", "severity": "high", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Execution" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/exfiltration_ec2_snapshot_change_activity.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/exfiltration_ec2_snapshot_change_activity.json index fc18a516be0f44..abf64d55ab99ab 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/exfiltration_ec2_snapshot_change_activity.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/exfiltration_ec2_snapshot_change_activity.json @@ -25,11 +25,12 @@ "rule_id": "98fd7407-0bd5-5817-cda0-3fcc33113a56", "severity": "medium", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Asset Visibility", - "Continuous Monitoring" + "Asset Visibility" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/exfiltration_gcp_logging_sink_modification.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/exfiltration_gcp_logging_sink_modification.json index 4e8954c3441cd0..5e3cc2da2f8712 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/exfiltration_gcp_logging_sink_modification.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/exfiltration_gcp_logging_sink_modification.json @@ -22,10 +22,11 @@ "severity": "low", "tags": [ "Elastic", + "Cloud", "GCP", "Continuous Monitoring", "SecOps", - "Logging" + "Log Auditing" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/external_alerts.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/external_alerts.json index 6bc14f4e5af8ab..a8a86348a10193 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/external_alerts.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/external_alerts.json @@ -54,9 +54,14 @@ } ], "tags": [ - "Elastic" + "Elastic", + "Network", + "Windows", + "APM", + "macOS", + "Linux" ], "timestamp_override": "event.ingested", "type": "query", - "version": 1 + "version": 2 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_attempt_to_revoke_okta_api_token.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_attempt_to_revoke_okta_api_token.json index f9d71a2e1cbffb..f2ad30fa260204 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_attempt_to_revoke_okta_api_token.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_attempt_to_revoke_okta_api_token.json @@ -24,10 +24,11 @@ "severity": "low", "tags": [ "Elastic", + "Identity", "Okta", + "Continuous Monitoring", "SecOps", - "Monitoring", - "Continuous Monitoring" + "Monitoring" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_azure_automation_runbook_deleted.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_azure_automation_runbook_deleted.json index 662709774f5baf..f474357cc6e2cf 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_azure_automation_runbook_deleted.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_azure_automation_runbook_deleted.json @@ -23,6 +23,7 @@ "severity": "low", "tags": [ "Elastic", + "Cloud", "Azure", "Continuous Monitoring", "SecOps", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_cloudtrail_logging_updated.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_cloudtrail_logging_updated.json index c648ae1ea4b5bf..d7f4c2b19bc0ff 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_cloudtrail_logging_updated.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_cloudtrail_logging_updated.json @@ -25,11 +25,12 @@ "rule_id": "3e002465-876f-4f04-b016-84ef48ce7e5d", "severity": "low", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Logging", - "Continuous Monitoring" + "Log Auditing" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_cloudwatch_log_group_deletion.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_cloudwatch_log_group_deletion.json index f7e9077c14314e..ae978f6564d670 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_cloudwatch_log_group_deletion.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_cloudwatch_log_group_deletion.json @@ -25,11 +25,12 @@ "rule_id": "68a7a5a5-a2fc-4a76-ba9f-26849de881b4", "severity": "medium", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Logging", - "Continuous Monitoring" + "Log Auditing" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_cloudwatch_log_stream_deletion.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_cloudwatch_log_stream_deletion.json index b50efb21e42f0f..7b985fdb6f693b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_cloudwatch_log_stream_deletion.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_cloudwatch_log_stream_deletion.json @@ -25,11 +25,12 @@ "rule_id": "d624f0ae-3dd1-4856-9aad-ccfe4d4bfa17", "severity": "medium", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Logging", - "Continuous Monitoring" + "Log Auditing" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_ec2_disable_ebs_encryption.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_ec2_disable_ebs_encryption.json index 370a65c31e7c17..c60619e8947175 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_ec2_disable_ebs_encryption.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_ec2_disable_ebs_encryption.json @@ -26,11 +26,12 @@ "rule_id": "bb9b13b2-1700-48a8-a750-b43b0a72ab69", "severity": "medium", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Data Protection", - "Continuous Monitoring" + "Data Protection" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_gcp_iam_role_deletion.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_gcp_iam_role_deletion.json index 2c67be7408d1dc..09a99966801552 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_gcp_iam_role_deletion.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_gcp_iam_role_deletion.json @@ -22,6 +22,7 @@ "severity": "low", "tags": [ "Elastic", + "Cloud", "GCP", "Continuous Monitoring", "SecOps", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_gcp_service_account_deleted.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_gcp_service_account_deleted.json index 2aa702e5ca4d13..9d34f31c1700db 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_gcp_service_account_deleted.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_gcp_service_account_deleted.json @@ -22,6 +22,7 @@ "severity": "medium", "tags": [ "Elastic", + "Cloud", "GCP", "Continuous Monitoring", "SecOps", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_gcp_service_account_disabled.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_gcp_service_account_disabled.json index 9b5188f43633de..606ebd1e6128e1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_gcp_service_account_disabled.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_gcp_service_account_disabled.json @@ -22,6 +22,7 @@ "severity": "medium", "tags": [ "Elastic", + "Cloud", "GCP", "Continuous Monitoring", "SecOps", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_gcp_storage_bucket_deleted.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_gcp_storage_bucket_deleted.json index 6adad4b687de7e..859c59ff8a3252 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_gcp_storage_bucket_deleted.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_gcp_storage_bucket_deleted.json @@ -22,6 +22,7 @@ "severity": "medium", "tags": [ "Elastic", + "Cloud", "GCP", "Continuous Monitoring", "SecOps", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_gcp_virtual_private_cloud_network_deleted.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_gcp_virtual_private_cloud_network_deleted.json index c5dc9f25f893fa..7f702f11a95151 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_gcp_virtual_private_cloud_network_deleted.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_gcp_virtual_private_cloud_network_deleted.json @@ -22,6 +22,7 @@ "severity": "medium", "tags": [ "Elastic", + "Cloud", "GCP", "Continuous Monitoring", "SecOps", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_gcp_virtual_private_cloud_route_created.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_gcp_virtual_private_cloud_route_created.json index 5e8fea09befc4c..1da90189f96b84 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_gcp_virtual_private_cloud_route_created.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_gcp_virtual_private_cloud_route_created.json @@ -23,6 +23,7 @@ "severity": "low", "tags": [ "Elastic", + "Cloud", "GCP", "Continuous Monitoring", "SecOps", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_gcp_virtual_private_cloud_route_deleted.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_gcp_virtual_private_cloud_route_deleted.json index 8482e0efbb036a..c379f07f021a6c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_gcp_virtual_private_cloud_route_deleted.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_gcp_virtual_private_cloud_route_deleted.json @@ -23,6 +23,7 @@ "severity": "medium", "tags": [ "Elastic", + "Cloud", "GCP", "Continuous Monitoring", "SecOps", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_hosts_file_modified.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_hosts_file_modified.json index bf04626dee2778..5d7e0bec4332c7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_hosts_file_modified.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_hosts_file_modified.json @@ -22,9 +22,12 @@ "severity": "medium", "tags": [ "Elastic", + "Host", "Linux", "Windows", - "macOS" + "macOS", + "Threat Detection", + "Impact" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_iam_deactivate_mfa_device.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_iam_deactivate_mfa_device.json index ca5cfd4ae596bb..83e184d9bf805d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_iam_deactivate_mfa_device.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_iam_deactivate_mfa_device.json @@ -25,11 +25,12 @@ "rule_id": "d8fc1cca-93ed-43c1-bbb6-c0dd3eff2958", "severity": "medium", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Monitoring", - "Continuous Monitoring" + "Monitoring" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_iam_group_deletion.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_iam_group_deletion.json index 00a10772d4d9a8..6212aa70747a5d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_iam_group_deletion.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_iam_group_deletion.json @@ -25,11 +25,12 @@ "rule_id": "867616ec-41e5-4edc-ada2-ab13ab45de8a", "severity": "low", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Monitoring", - "Continuous Monitoring" + "Monitoring" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_possible_okta_dos_attack.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_possible_okta_dos_attack.json index 9bc44bf4e6da93..d1852478c666fa 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_possible_okta_dos_attack.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_possible_okta_dos_attack.json @@ -21,10 +21,11 @@ "severity": "medium", "tags": [ "Elastic", + "Identity", "Okta", + "Continuous Monitoring", "SecOps", - "Monitoring", - "Continuous Monitoring" + "Monitoring" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_rds_cluster_deletion.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_rds_cluster_deletion.json index 829c244dd45c0c..116483173564ed 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_rds_cluster_deletion.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_rds_cluster_deletion.json @@ -27,11 +27,12 @@ "rule_id": "9055ece6-2689-4224-a0e0-b04881e1f8ad", "severity": "medium", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Asset Visibility", - "Continuous Monitoring" + "Asset Visibility" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_rds_instance_cluster_stoppage.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_rds_instance_cluster_stoppage.json index 68459d0e777b91..32276491493805 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_rds_instance_cluster_stoppage.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_rds_instance_cluster_stoppage.json @@ -27,11 +27,12 @@ "rule_id": "ecf2b32c-e221-4bd4-aa3b-c7d59b3bc01d", "severity": "medium", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Asset Visibility", - "Continuous Monitoring" + "Asset Visibility" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_resource_group_deletion.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_resource_group_deletion.json index a0c56c19b964e9..8086c09e4b1742 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_resource_group_deletion.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/impact_resource_group_deletion.json @@ -23,10 +23,11 @@ "severity": "medium", "tags": [ "Elastic", + "Cloud", "Azure", - "SecOps", "Continuous Monitoring", - "Logging" + "SecOps", + "Log Auditing" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/index.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/index.ts index 6e376930617def..5fec97e83bad41 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/index.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/index.ts @@ -221,14 +221,14 @@ import rule209 from './credential_access_okta_brute_force_or_password_spraying.j import rule210 from './execution_unusual_dns_service_children.json'; import rule211 from './execution_unusual_dns_service_file_writes.json'; import rule212 from './lateral_movement_dns_server_overflow.json'; -import rule213 from './initial_access_root_console_failure_brute_force.json'; +import rule213 from './credential_access_root_console_failure_brute_force.json'; import rule214 from './initial_access_unsecure_elasticsearch_node.json'; import rule215 from './credential_access_domain_backup_dpapi_private_keys.json'; -import rule216 from './lateral_movement_gpo_schtask_service_creation.json'; -import rule217 from './credential_access_kerberosdump_kcc.json'; -import rule218 from './defense_evasion_execution_suspicious_psexesvc.json'; -import rule219 from './execution_via_xp_cmdshell_mssql_stored_procedure.json'; -import rule220 from './exfiltration_compress_credentials_keychains.json'; +import rule216 from './persistence_gpo_schtask_service_creation.json'; +import rule217 from './credential_access_compress_credentials_keychains.json'; +import rule218 from './credential_access_kerberosdump_kcc.json'; +import rule219 from './execution_suspicious_psexesvc.json'; +import rule220 from './execution_via_xp_cmdshell_mssql_stored_procedure.json'; import rule221 from './privilege_escalation_printspooler_service_suspicious_file.json'; import rule222 from './privilege_escalation_printspooler_suspicious_spl_file.json'; import rule223 from './defense_evasion_azure_diagnostic_settings_deletion.json'; @@ -250,16 +250,16 @@ import rule238 from './persistence_azure_automation_webhook_created.json'; import rule239 from './privilege_escalation_uac_bypass_diskcleanup_hijack.json'; import rule240 from './credential_access_attempts_to_brute_force_okta_user_account.json'; import rule241 from './credential_access_storage_account_key_regenerated.json'; -import rule242 from './credential_access_suspicious_okta_user_password_reset_or_unlock_attempts.json'; +import rule242 from './defense_evasion_suspicious_okta_user_password_reset_or_unlock_attempts.json'; import rule243 from './defense_evasion_system_critical_proc_abnormal_file_activity.json'; import rule244 from './defense_evasion_unusual_system_vp_child_program.json'; -import rule245 from './defense_evasion_mfa_disabled_for_azure_user.json'; -import rule246 from './discovery_blob_container_access_mod.json'; +import rule245 from './discovery_blob_container_access_mod.json'; +import rule246 from './persistence_mfa_disabled_for_azure_user.json'; import rule247 from './persistence_user_added_as_owner_for_azure_application.json'; import rule248 from './persistence_user_added_as_owner_for_azure_service_principal.json'; -import rule249 from './defense_evasion_suspicious_managedcode_host_process.json'; -import rule250 from './execution_command_shell_started_by_unusual_process.json'; -import rule251 from './execution_suspicious_dotnet_compiler_parent_process.json'; +import rule249 from './defense_evasion_dotnet_compiler_parent_process.json'; +import rule250 from './defense_evasion_suspicious_managedcode_host_process.json'; +import rule251 from './execution_command_shell_started_by_unusual_process.json'; import rule252 from './defense_evasion_masquerading_as_elastic_endpoint_process.json'; import rule253 from './defense_evasion_masquerading_suspicious_werfault_childproc.json'; import rule254 from './defense_evasion_masquerading_werfault.json'; @@ -268,28 +268,28 @@ import rule256 from './credential_access_mimikatz_memssp_default_logs.json'; import rule257 from './defense_evasion_code_injection_conhost.json'; import rule258 from './defense_evasion_network_watcher_deletion.json'; import rule259 from './initial_access_external_guest_user_invite.json'; -import rule260 from './defense_evasion_azure_conditional_access_policy_modified.json'; -import rule261 from './defense_evasion_azure_privileged_identity_management_role_modified.json'; -import rule262 from './defense_evasion_masquerading_renamed_autoit.json'; -import rule263 from './impact_azure_automation_runbook_deleted.json'; -import rule264 from './initial_access_consent_grant_attack_via_azure_registered_application.json'; -import rule265 from './c2_installutil_beacon.json'; -import rule266 from './c2_msbuild_beacon_sequence.json'; -import rule267 from './c2_mshta_beacon.json'; -import rule268 from './c2_msxsl_beacon.json'; -import rule269 from './c2_network_connection_from_windows_binary.json'; -import rule270 from './c2_reg_beacon.json'; -import rule271 from './c2_rundll32_sequence.json'; -import rule272 from './command_and_control_teamviewer_remote_file_copy.json'; -import rule273 from './escalation_uac_sdclt.json'; -import rule274 from './evasion_rundll32_no_arguments.json'; -import rule275 from './evasion_suspicious_scrobj_load.json'; -import rule276 from './evasion_suspicious_wmi_script.json'; -import rule277 from './execution_ms_office_written_file.json'; -import rule278 from './execution_pdf_written_file.json'; -import rule279 from './execution_wpad_exploitation.json'; -import rule280 from './lateral_movement_cmd_service.json'; -import rule281 from './persistence_app_compat_shim.json'; +import rule260 from './defense_evasion_masquerading_renamed_autoit.json'; +import rule261 from './impact_azure_automation_runbook_deleted.json'; +import rule262 from './initial_access_consent_grant_attack_via_azure_registered_application.json'; +import rule263 from './persistence_azure_conditional_access_policy_modified.json'; +import rule264 from './persistence_azure_privileged_identity_management_role_modified.json'; +import rule265 from './command_and_control_teamviewer_remote_file_copy.json'; +import rule266 from './defense_evasion_installutil_beacon.json'; +import rule267 from './defense_evasion_msbuild_beacon_sequence.json'; +import rule268 from './defense_evasion_mshta_beacon.json'; +import rule269 from './defense_evasion_msxsl_beacon.json'; +import rule270 from './defense_evasion_network_connection_from_windows_binary.json'; +import rule271 from './defense_evasion_reg_beacon.json'; +import rule272 from './defense_evasion_rundll32_no_arguments.json'; +import rule273 from './defense_evasion_rundll32_sequence.json'; +import rule274 from './defense_evasion_suspicious_scrobj_load.json'; +import rule275 from './defense_evasion_suspicious_wmi_script.json'; +import rule276 from './execution_ms_office_written_file.json'; +import rule277 from './execution_pdf_written_file.json'; +import rule278 from './execution_wpad_exploitation.json'; +import rule279 from './lateral_movement_cmd_service.json'; +import rule280 from './persistence_app_compat_shim.json'; +import rule281 from './privilege_escalation_uac_sdclt.json'; import rule282 from './command_and_control_remote_file_copy_desktopimgdownldr.json'; import rule283 from './command_and_control_remote_file_copy_mpcmdrun.json'; import rule284 from './defense_evasion_execution_suspicious_explorer_winword.json'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_consent_grant_attack_via_azure_registered_application.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_consent_grant_attack_via_azure_registered_application.json index 8147859fa4e6f0..1dab4e8df71b44 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_consent_grant_attack_via_azure_registered_application.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_consent_grant_attack_via_azure_registered_application.json @@ -20,6 +20,7 @@ "severity": "medium", "tags": [ "Elastic", + "Cloud", "Azure", "Continuous Monitoring", "SecOps", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_console_login_root.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_console_login_root.json index 621881e2641382..1d6f5b2c90f260 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_console_login_root.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_console_login_root.json @@ -24,11 +24,12 @@ "rule_id": "e2a67480-3b79-403d-96e3-fdd2992c50ef", "severity": "high", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Identity and Access", - "Continuous Monitoring" + "Identity and Access" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_external_guest_user_invite.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_external_guest_user_invite.json index 392e0ec745fc2b..455fc3c7629784 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_external_guest_user_invite.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_external_guest_user_invite.json @@ -23,9 +23,10 @@ "severity": "low", "tags": [ "Elastic", + "Cloud", "Azure", - "SecOps", "Continuous Monitoring", + "SecOps", "Identity and Access" ], "threat": [ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_gcp_iam_custom_role_creation.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_gcp_iam_custom_role_creation.json index 0eab41ad8c4bda..ff7ad0e8d29a27 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_gcp_iam_custom_role_creation.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_gcp_iam_custom_role_creation.json @@ -22,6 +22,7 @@ "severity": "medium", "tags": [ "Elastic", + "Cloud", "GCP", "Continuous Monitoring", "SecOps", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_password_recovery.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_password_recovery.json index 2f0eed31d05bec..13279ea1703585 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_password_recovery.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_password_recovery.json @@ -24,11 +24,12 @@ "rule_id": "69c420e8-6c9e-4d28-86c0-8a2be2d1e78c", "severity": "low", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Identity and Access", - "Continuous Monitoring" + "Identity and Access" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_rdp_remote_desktop_protocol_to_the_internet.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_rdp_remote_desktop_protocol_to_the_internet.json index 15c3c81a551bdb..ce0f44713523f5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_rdp_remote_desktop_protocol_to_the_internet.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_rdp_remote_desktop_protocol_to_the_internet.json @@ -8,7 +8,8 @@ ], "index": [ "filebeat-*", - "packetbeat-*" + "packetbeat-*", + "logs-endpoint.events.*" ], "language": "kuery", "license": "Elastic License", @@ -19,7 +20,10 @@ "severity": "low", "tags": [ "Elastic", - "Network" + "Host", + "Network", + "Threat Detection", + "Initial Access" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_rpc_remote_procedure_call_from_the_internet.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_rpc_remote_procedure_call_from_the_internet.json index 7c61f95f9e9f3d..b8f3e01823312c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_rpc_remote_procedure_call_from_the_internet.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_rpc_remote_procedure_call_from_the_internet.json @@ -5,7 +5,8 @@ "description": "This rule detects network events that may indicate the use of RPC traffic from the Internet. RPC is commonly used by system administrators to remotely control a system for maintenance or to use shared resources. It should almost never be directly exposed to the Internet, as it is frequently targeted and exploited by threat actors as an initial access or back-door vector.", "index": [ "filebeat-*", - "packetbeat-*" + "packetbeat-*", + "logs-endpoint.events.*" ], "language": "kuery", "license": "Elastic License", @@ -16,7 +17,10 @@ "severity": "high", "tags": [ "Elastic", - "Network" + "Host", + "Network", + "Threat Detection", + "Initial Access" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_rpc_remote_procedure_call_to_the_internet.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_rpc_remote_procedure_call_to_the_internet.json index e8da93ed9d1c7c..e8e4ea4eb37466 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_rpc_remote_procedure_call_to_the_internet.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_rpc_remote_procedure_call_to_the_internet.json @@ -5,7 +5,8 @@ "description": "This rule detects network events that may indicate the use of RPC traffic to the Internet. RPC is commonly used by system administrators to remotely control a system for maintenance or to use shared resources. It should almost never be directly exposed to the Internet, as it is frequently targeted and exploited by threat actors as an initial access or back-door vector.", "index": [ "filebeat-*", - "packetbeat-*" + "packetbeat-*", + "logs-endpoint.events.*" ], "language": "kuery", "license": "Elastic License", @@ -16,7 +17,10 @@ "severity": "high", "tags": [ "Elastic", - "Network" + "Host", + "Network", + "Threat Detection", + "Initial Access" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_smb_windows_file_sharing_activity_to_the_internet.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_smb_windows_file_sharing_activity_to_the_internet.json index aff8a415b7e352..fec0f308a8d276 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_smb_windows_file_sharing_activity_to_the_internet.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_smb_windows_file_sharing_activity_to_the_internet.json @@ -5,7 +5,8 @@ "description": "This rule detects network events that may indicate the use of Windows file sharing (also called SMB or CIFS) traffic to the Internet. SMB is commonly used within networks to share files, printers, and other system resources amongst trusted systems. It should almost never be directly exposed to the Internet, as it is frequently targeted and exploited by threat actors as an initial access or back-door vector or for data exfiltration.", "index": [ "filebeat-*", - "packetbeat-*" + "packetbeat-*", + "logs-endpoint.events.*" ], "language": "kuery", "license": "Elastic License", @@ -16,7 +17,10 @@ "severity": "high", "tags": [ "Elastic", - "Network" + "Host", + "Network", + "Threat Detection", + "Initial Access" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_suspicious_activity_reported_by_okta_user.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_suspicious_activity_reported_by_okta_user.json index 24837084c83810..5b1946dc7c07d3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_suspicious_activity_reported_by_okta_user.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_suspicious_activity_reported_by_okta_user.json @@ -24,10 +24,11 @@ "severity": "medium", "tags": [ "Elastic", + "Identity", "Okta", + "Continuous Monitoring", "SecOps", - "Monitoring", - "Continuous Monitoring" + "Monitoring" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_unsecure_elasticsearch_node.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_unsecure_elasticsearch_node.json index e6d718a23eb965..a577a3413e5df6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_unsecure_elasticsearch_node.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/initial_access_unsecure_elasticsearch_node.json @@ -23,7 +23,9 @@ "severity": "medium", "tags": [ "Elastic", - "Network" + "Network", + "Threat Detection", + "Initial Access" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_cmd_service.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_cmd_service.json index bd14db77b9fe9d..f4e137663762ac 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_cmd_service.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_cmd_service.json @@ -11,13 +11,16 @@ "language": "eql", "license": "Elastic License", "name": "Service Command Lateral Movement", - "query": "/* dependent on a wildcard for remote path */\n\nsequence by process.entity_id with maxspan=1m\n [process where event.type in (\"start\", \"process_started\") and\n (process.name == \"sc.exe\" or process.pe.original_file_name == \"sc.exe\") and\n wildcard(process.args, \"\\\\\\\\*\") and wildcard(process.args, \"binPath*\", \"binpath*\") and\n process.args in (\"create\", \"config\", \"failure\", \"start\")]\n [network where event.type == \"connection\" and process.name == \"sc.exe\" and destination.address != \"127.0.0.1\"]\n", + "query": "sequence by process.entity_id with maxspan=1m\n [process where event.type in (\"start\", \"process_started\") and\n /* uncomment once in winlogbeat */\n (process.name == \"sc.exe\" /* or process.pe.original_file_name == \"sc.exe\" */ ) and\n /* case insensitive */\n wildcard(process.args, \"\\\\\\\\*\") and wildcard(process.args, \"binPath=*\", \"binpath=*\") and \n (process.args : \"create\" or\n process.args : \"config\" or\n process.args : \"failure\" or\n process.args : \"start\")]\n [network where event.type == \"connection\" and process.name : \"sc.exe\" and destination.ip != \"127.0.0.1\"]\n", "risk_score": 21, "rule_id": "d61cbcf8-1bc1-4cff-85ba-e7b21c5beedc", "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Lateral Movement" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_direct_outbound_smb_connection.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_direct_outbound_smb_connection.json index e4014b22a6c091..39f45a736383a0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_direct_outbound_smb_connection.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_direct_outbound_smb_connection.json @@ -17,7 +17,10 @@ "severity": "medium", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Lateral Movement" ], "threat": [ { @@ -37,5 +40,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_dns_server_overflow.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_dns_server_overflow.json index 2a86dcac12e7bd..052d0de3ef79e7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_dns_server_overflow.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_dns_server_overflow.json @@ -26,7 +26,8 @@ "tags": [ "Elastic", "Network", - "Windows" + "Threat Detection", + "Lateral Movement" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_remote_ssh_login_enabled.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_remote_ssh_login_enabled.json index f1ce68abf8302b..149dab112929c1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_remote_ssh_login_enabled.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_remote_ssh_login_enabled.json @@ -21,7 +21,10 @@ "severity": "medium", "tags": [ "Elastic", - "MacOS" + "Host", + "macOS", + "Threat Detection", + "Lateral Movement" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_telnet_network_activity_external.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_telnet_network_activity_external.json index e4804329c0f307..ff62251a465f77 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_telnet_network_activity_external.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_telnet_network_activity_external.json @@ -20,7 +20,10 @@ "severity": "medium", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection", + "Lateral Movement" ], "threat": [ { @@ -40,5 +43,5 @@ } ], "type": "query", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_telnet_network_activity_internal.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_telnet_network_activity_internal.json index 30312987d166c3..bb6c03e9410e57 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_telnet_network_activity_internal.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_telnet_network_activity_internal.json @@ -20,7 +20,10 @@ "severity": "medium", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection", + "Lateral Movement" ], "threat": [ { @@ -40,5 +43,5 @@ } ], "type": "query", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_hping_activity.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_hping_activity.json index 3a5c4d9e69d496..775d201cd18083 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_hping_activity.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_hping_activity.json @@ -23,8 +23,10 @@ "severity": "high", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection" ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_iodine_activity.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_iodine_activity.json index 63c82c5662df66..510eecb9d2fee4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_iodine_activity.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_iodine_activity.json @@ -23,8 +23,10 @@ "severity": "high", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection" ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_mknod_activity.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_mknod_activity.json index 99d087fe675a67..a81567740590d6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_mknod_activity.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_mknod_activity.json @@ -23,7 +23,9 @@ "severity": "low", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection" ], "type": "query", "version": 5 diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_netcat_network_connection.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_netcat_network_connection.json index bce10f640691b7..ebe884debc9a0f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_netcat_network_connection.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_netcat_network_connection.json @@ -25,8 +25,10 @@ "severity": "medium", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection" ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_nmap_activity.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_nmap_activity.json index 5d9e338425bdae..76866f67d72888 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_nmap_activity.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_nmap_activity.json @@ -23,8 +23,10 @@ "severity": "low", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection" ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_nping_activity.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_nping_activity.json index bd019c9a80c4c5..cc744614d68af4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_nping_activity.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_nping_activity.json @@ -23,8 +23,10 @@ "severity": "medium", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection" ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_process_started_in_temp_directory.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_process_started_in_temp_directory.json index f0bbc892d7d9c4..1662d63066feb3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_process_started_in_temp_directory.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_process_started_in_temp_directory.json @@ -20,8 +20,10 @@ "severity": "medium", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection" ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_socat_activity.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_socat_activity.json index fac03d31b57bfe..07cd9e4a065b33 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_socat_activity.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_socat_activity.json @@ -23,8 +23,10 @@ "severity": "medium", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection" ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_strace_activity.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_strace_activity.json index c1b782d612ccbc..06be108db2f140 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_strace_activity.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/linux_strace_activity.json @@ -23,8 +23,10 @@ "severity": "low", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection" ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_cloudtrail_error_message_spike.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_cloudtrail_error_message_spike.json index 0730c421cf5f2e..890b777751a038 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_cloudtrail_error_message_spike.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_cloudtrail_error_message_spike.json @@ -20,10 +20,11 @@ "rule_id": "78d3d8d9-b476-451d-a9e0-7a5addd70670", "severity": "low", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", "ML" ], "type": "machine_learning", - "version": 1 + "version": 2 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_cloudtrail_rare_error_code.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_cloudtrail_rare_error_code.json index 8003cdd7504c76..62254d970ca7e1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_cloudtrail_rare_error_code.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_cloudtrail_rare_error_code.json @@ -20,10 +20,11 @@ "rule_id": "19de8096-e2b0-4bd8-80c9-34a820813fff", "severity": "low", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", "ML" ], "type": "machine_learning", - "version": 1 + "version": 2 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_cloudtrail_rare_method_by_city.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_cloudtrail_rare_method_by_city.json index 2c54dbd03daba5..5c1189c989bac3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_cloudtrail_rare_method_by_city.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_cloudtrail_rare_method_by_city.json @@ -20,10 +20,11 @@ "rule_id": "809b70d3-e2c3-455e-af1b-2626a5a1a276", "severity": "low", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", "ML" ], "type": "machine_learning", - "version": 1 + "version": 2 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_cloudtrail_rare_method_by_country.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_cloudtrail_rare_method_by_country.json index 68cbf4979a933f..149ebd1a84fd4f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_cloudtrail_rare_method_by_country.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_cloudtrail_rare_method_by_country.json @@ -20,10 +20,11 @@ "rule_id": "dca28dee-c999-400f-b640-50a081cc0fd1", "severity": "low", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", "ML" ], "type": "machine_learning", - "version": 1 + "version": 2 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_cloudtrail_rare_method_by_user.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_cloudtrail_rare_method_by_user.json index e4ec651e719346..a8c0bce1694324 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_cloudtrail_rare_method_by_user.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_cloudtrail_rare_method_by_user.json @@ -20,10 +20,11 @@ "rule_id": "ac706eae-d5ec-4b14-b4fd-e8ba8086f0e1", "severity": "low", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", "ML" ], "type": "machine_learning", - "version": 1 + "version": 2 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_compiler_activity.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_compiler_activity.json index eb764c5e40817c..592bd723e1a0ca 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_compiler_activity.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_compiler_activity.json @@ -17,7 +17,9 @@ "severity": "low", "tags": [ "Elastic", + "Host", "Linux", + "Threat Detection", "ML" ], "type": "machine_learning", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_kernel_module_arguments.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_kernel_module_arguments.json index d289e0ba6f0084..52a1d6dd5c60a7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_kernel_module_arguments.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_kernel_module_arguments.json @@ -20,7 +20,9 @@ "severity": "low", "tags": [ "Elastic", + "Host", "Linux", + "Threat Detection", "ML" ], "threat": [ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_metadata_process.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_metadata_process.json index c1cc619164b1fc..ec033a609cac1c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_metadata_process.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_metadata_process.json @@ -11,13 +11,15 @@ "interval": "15m", "license": "Elastic License", "machine_learning_job_id": "linux_rare_metadata_process", - "name": "Unusual Process Calling the Metadata Service", + "name": "Unusual Linux Process Calling the Metadata Service", "risk_score": 21, "rule_id": "9d302377-d226-4e12-b54c-1906b5aec4f6", "severity": "low", "tags": [ "Elastic", + "Host", "Linux", + "Threat Detection", "ML" ], "type": "machine_learning", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_metadata_user.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_metadata_user.json index 59a04dd54dd892..a4a6b38bd32b84 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_metadata_user.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_metadata_user.json @@ -17,7 +17,9 @@ "severity": "low", "tags": [ "Elastic", + "Host", "Linux", + "Threat Detection", "ML" ], "type": "machine_learning", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_network_activity.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_network_activity.json index bf86f78fe3e72e..04df4adab85258 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_network_activity.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_network_activity.json @@ -18,9 +18,11 @@ "severity": "low", "tags": [ "Elastic", + "Host", "Linux", + "Threat Detection", "ML" ], "type": "machine_learning", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_network_port_activity.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_network_port_activity.json index a588a6f5bcb0a5..c5a0891a7bbc48 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_network_port_activity.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_network_port_activity.json @@ -20,9 +20,11 @@ "severity": "low", "tags": [ "Elastic", + "Host", "Linux", + "Threat Detection", "ML" ], "type": "machine_learning", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_network_service.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_network_service.json index 5c56845024eb29..81dff605dc779b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_network_service.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_network_service.json @@ -20,9 +20,11 @@ "severity": "low", "tags": [ "Elastic", + "Host", "Linux", + "Threat Detection", "ML" ], "type": "machine_learning", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_network_url_activity.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_network_url_activity.json index 3b3f751dfc60b0..5336c31d4a7414 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_network_url_activity.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_network_url_activity.json @@ -20,9 +20,11 @@ "severity": "low", "tags": [ "Elastic", + "Host", "Linux", + "Threat Detection", "ML" ], "type": "machine_learning", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_process_all_hosts.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_process_all_hosts.json index 8475410735f34f..04d112c26e8869 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_process_all_hosts.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_process_all_hosts.json @@ -21,9 +21,11 @@ "severity": "low", "tags": [ "Elastic", + "Host", "Linux", + "Threat Detection", "ML" ], "type": "machine_learning", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_sudo_activity.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_sudo_activity.json index 8f03b24a6bd181..57d5260d111dbf 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_sudo_activity.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_sudo_activity.json @@ -17,7 +17,9 @@ "severity": "low", "tags": [ "Elastic", + "Host", "Linux", + "Threat Detection", "ML" ], "threat": [ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_user_name.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_user_name.json index 3e4b1f15fdce4d..3f889dc5293170 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_user_name.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_anomalous_user_name.json @@ -21,9 +21,11 @@ "severity": "low", "tags": [ "Elastic", + "Host", "Linux", + "Threat Detection", "ML" ], "type": "machine_learning", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_system_information_discovery.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_system_information_discovery.json index 40f117c6a5708c..cdd1ee8d466539 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_system_information_discovery.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_system_information_discovery.json @@ -17,7 +17,9 @@ "severity": "low", "tags": [ "Elastic", + "Host", "Linux", + "Threat Detection", "ML" ], "threat": [ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_system_network_configuration_discovery.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_system_network_configuration_discovery.json index 326024114f1458..b143293ac995b2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_system_network_configuration_discovery.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_system_network_configuration_discovery.json @@ -17,7 +17,9 @@ "severity": "low", "tags": [ "Elastic", + "Host", "Linux", + "Threat Detection", "ML" ], "threat": [ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_system_network_connection_discovery.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_system_network_connection_discovery.json index 881a2f9fa34104..ffab1c9eb84950 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_system_network_connection_discovery.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_system_network_connection_discovery.json @@ -17,7 +17,9 @@ "severity": "low", "tags": [ "Elastic", + "Host", "Linux", + "Threat Detection", "ML" ], "threat": [ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_system_process_discovery.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_system_process_discovery.json index 66859e2f9ccbf4..69d29400290110 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_system_process_discovery.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_system_process_discovery.json @@ -17,7 +17,9 @@ "severity": "low", "tags": [ "Elastic", + "Host", "Linux", + "Threat Detection", "ML" ], "threat": [ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_system_user_discovery.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_system_user_discovery.json index 4437334b0aa1f9..b5c5e66af6e7e8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_system_user_discovery.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_linux_system_user_discovery.json @@ -17,7 +17,9 @@ "severity": "low", "tags": [ "Elastic", + "Host", "Linux", + "Threat Detection", "ML" ], "threat": [ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_packetbeat_dns_tunneling.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_packetbeat_dns_tunneling.json index 1352fde91b59b7..fbce00ebaf5a36 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_packetbeat_dns_tunneling.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_packetbeat_dns_tunneling.json @@ -20,9 +20,10 @@ "severity": "low", "tags": [ "Elastic", - "ML", - "Packetbeat" + "Network", + "Threat Detection", + "ML" ], "type": "machine_learning", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_packetbeat_rare_dns_question.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_packetbeat_rare_dns_question.json index b16e67052a2122..744e9fbfc18d81 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_packetbeat_rare_dns_question.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_packetbeat_rare_dns_question.json @@ -20,9 +20,10 @@ "severity": "low", "tags": [ "Elastic", - "ML", - "Packetbeat" + "Network", + "Threat Detection", + "ML" ], "type": "machine_learning", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_packetbeat_rare_server_domain.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_packetbeat_rare_server_domain.json index a8971300fe11b7..a6729856558595 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_packetbeat_rare_server_domain.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_packetbeat_rare_server_domain.json @@ -20,9 +20,10 @@ "severity": "low", "tags": [ "Elastic", - "ML", - "Packetbeat" + "Network", + "Threat Detection", + "ML" ], "type": "machine_learning", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_packetbeat_rare_urls.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_packetbeat_rare_urls.json index 469f5d741ef6e4..5600fbe74c7fc7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_packetbeat_rare_urls.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_packetbeat_rare_urls.json @@ -20,9 +20,10 @@ "severity": "low", "tags": [ "Elastic", - "ML", - "Packetbeat" + "Network", + "Threat Detection", + "ML" ], "type": "machine_learning", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_packetbeat_rare_user_agent.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_packetbeat_rare_user_agent.json index ebcf4f987e9ded..5cce9c63fd88ed 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_packetbeat_rare_user_agent.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_packetbeat_rare_user_agent.json @@ -20,9 +20,10 @@ "severity": "low", "tags": [ "Elastic", - "ML", - "Packetbeat" + "Network", + "Threat Detection", + "ML" ], "type": "machine_learning", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_rare_process_by_host_linux.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_rare_process_by_host_linux.json index 385158dd6b65de..5c25b47833112d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_rare_process_by_host_linux.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_rare_process_by_host_linux.json @@ -21,9 +21,11 @@ "severity": "low", "tags": [ "Elastic", + "Host", "Linux", + "Threat Detection", "ML" ], "type": "machine_learning", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_rare_process_by_host_windows.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_rare_process_by_host_windows.json index d0a99b32d47132..e86bc96fddae59 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_rare_process_by_host_windows.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_rare_process_by_host_windows.json @@ -21,9 +21,11 @@ "severity": "low", "tags": [ "Elastic", - "ML", - "Windows" + "Host", + "Windows", + "Threat Detection", + "ML" ], "type": "machine_learning", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_suspicious_login_activity.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_suspicious_login_activity.json index f309debcdffe95..ae0ac2b716e323 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_suspicious_login_activity.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_suspicious_login_activity.json @@ -20,9 +20,11 @@ "severity": "low", "tags": [ "Elastic", + "Host", "Linux", + "Threat Detection", "ML" ], "type": "machine_learning", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_metadata_process.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_metadata_process.json index 56874ec371b435..3e35b2af5e3b82 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_metadata_process.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_metadata_process.json @@ -17,8 +17,10 @@ "severity": "low", "tags": [ "Elastic", - "ML", - "Windows" + "Host", + "Windows", + "Threat Detection", + "ML" ], "type": "machine_learning", "version": 1 diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_metadata_user.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_metadata_user.json index f124cda7717c39..7c45ab4dfa08ba 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_metadata_user.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_metadata_user.json @@ -17,8 +17,10 @@ "severity": "low", "tags": [ "Elastic", - "ML", - "Windows" + "Host", + "Windows", + "Threat Detection", + "ML" ], "type": "machine_learning", "version": 1 diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_network_activity.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_network_activity.json index 0ab591097f975c..937fcbc079eb8a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_network_activity.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_network_activity.json @@ -21,9 +21,11 @@ "severity": "low", "tags": [ "Elastic", - "ML", - "Windows" + "Host", + "Windows", + "Threat Detection", + "ML" ], "type": "machine_learning", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_path_activity.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_path_activity.json index a7b309e6d7fcdb..d45661f737b5dd 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_path_activity.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_path_activity.json @@ -20,9 +20,11 @@ "severity": "low", "tags": [ "Elastic", - "ML", - "Windows" + "Host", + "Windows", + "Threat Detection", + "ML" ], "type": "machine_learning", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_process_all_hosts.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_process_all_hosts.json index bc6346f457b651..88764ed1bf46d5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_process_all_hosts.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_process_all_hosts.json @@ -21,9 +21,11 @@ "severity": "low", "tags": [ "Elastic", - "ML", - "Windows" + "Host", + "Windows", + "Threat Detection", + "ML" ], "type": "machine_learning", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_process_creation.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_process_creation.json index 97351a1f517b35..2ebc7ee667d4e4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_process_creation.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_process_creation.json @@ -20,9 +20,11 @@ "severity": "low", "tags": [ "Elastic", - "ML", - "Windows" + "Host", + "Windows", + "Threat Detection", + "ML" ], "type": "machine_learning", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_script.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_script.json index d0dc8d7e40fa21..91714b10d2ab6b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_script.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_script.json @@ -20,9 +20,11 @@ "severity": "low", "tags": [ "Elastic", - "ML", - "Windows" + "Host", + "Windows", + "Threat Detection", + "ML" ], "type": "machine_learning", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_service.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_service.json index b7e7a0357e1185..3aea73eb489e5a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_service.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_service.json @@ -20,9 +20,11 @@ "severity": "low", "tags": [ "Elastic", - "ML", - "Windows" + "Host", + "Windows", + "Threat Detection", + "ML" ], "type": "machine_learning", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_user_name.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_user_name.json index 26bd6837cbde5b..749cbb218bcd84 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_user_name.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_anomalous_user_name.json @@ -21,9 +21,11 @@ "severity": "low", "tags": [ "Elastic", - "ML", - "Windows" + "Host", + "Windows", + "Threat Detection", + "ML" ], "type": "machine_learning", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_rare_user_runas_event.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_rare_user_runas_event.json index 1af765f568bb1d..8ed8a005d9aa54 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_rare_user_runas_event.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_rare_user_runas_event.json @@ -20,9 +20,11 @@ "severity": "low", "tags": [ "Elastic", - "ML", - "Windows" + "Host", + "Windows", + "Threat Detection", + "ML" ], "type": "machine_learning", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_rare_user_type10_remote_login.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_rare_user_type10_remote_login.json index b69e759120ce4d..dda4e0bbad88d8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_rare_user_type10_remote_login.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/ml_windows_rare_user_type10_remote_login.json @@ -21,9 +21,11 @@ "severity": "low", "tags": [ "Elastic", - "ML", - "Windows" + "Host", + "Windows", + "Threat Detection", + "ML" ], "type": "machine_learning", - "version": 2 + "version": 3 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/okta_attempt_to_deactivate_okta_mfa_rule.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/okta_attempt_to_deactivate_okta_mfa_rule.json index c503d2298adad6..0ee0bbd6d62267 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/okta_attempt_to_deactivate_okta_mfa_rule.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/okta_attempt_to_deactivate_okta_mfa_rule.json @@ -24,10 +24,11 @@ "severity": "low", "tags": [ "Elastic", + "Identity", "Okta", + "Continuous Monitoring", "SecOps", - "Identity and Access", - "Continuous Monitoring" + "Identity and Access" ], "type": "query", "version": 2 diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/okta_attempt_to_delete_okta_policy.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/okta_attempt_to_delete_okta_policy.json index d095d7c1166de6..211fdb1ae3474f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/okta_attempt_to_delete_okta_policy.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/okta_attempt_to_delete_okta_policy.json @@ -24,10 +24,11 @@ "severity": "low", "tags": [ "Elastic", + "Identity", "Okta", + "Continuous Monitoring", "SecOps", - "Monitoring", - "Continuous Monitoring" + "Monitoring" ], "type": "query", "version": 2 diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/okta_attempt_to_modify_okta_mfa_rule.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/okta_attempt_to_modify_okta_mfa_rule.json index 2fe27575b7b2ae..eb726e24c89dab 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/okta_attempt_to_modify_okta_mfa_rule.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/okta_attempt_to_modify_okta_mfa_rule.json @@ -24,10 +24,11 @@ "severity": "low", "tags": [ "Elastic", + "Identity", "Okta", + "Continuous Monitoring", "SecOps", - "Identity and Access", - "Continuous Monitoring" + "Identity and Access" ], "type": "query", "version": 2 diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/okta_attempt_to_modify_okta_network_zone.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/okta_attempt_to_modify_okta_network_zone.json index 23b3313488847e..682dc17f0ed497 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/okta_attempt_to_modify_okta_network_zone.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/okta_attempt_to_modify_okta_network_zone.json @@ -24,10 +24,11 @@ "severity": "medium", "tags": [ "Elastic", + "Identity", "Okta", + "Continuous Monitoring", "SecOps", - "Network", - "Continuous Monitoring" + "Network Security" ], "type": "query", "version": 2 diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/okta_attempt_to_modify_okta_policy.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/okta_attempt_to_modify_okta_policy.json index 5b19031046b666..88e556d37a27cf 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/okta_attempt_to_modify_okta_policy.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/okta_attempt_to_modify_okta_policy.json @@ -24,10 +24,11 @@ "severity": "low", "tags": [ "Elastic", + "Identity", "Okta", + "Continuous Monitoring", "SecOps", - "Monitoring", - "Continuous Monitoring" + "Monitoring" ], "type": "query", "version": 2 diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/okta_attempt_to_modify_or_delete_application_sign_on_policy.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/okta_attempt_to_modify_or_delete_application_sign_on_policy.json index 58ba13e147a381..262a91f8e25c97 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/okta_attempt_to_modify_or_delete_application_sign_on_policy.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/okta_attempt_to_modify_or_delete_application_sign_on_policy.json @@ -24,10 +24,11 @@ "severity": "medium", "tags": [ "Elastic", + "Identity", "Okta", + "Continuous Monitoring", "SecOps", - "Identity and Access", - "Continuous Monitoring" + "Identity and Access" ], "type": "query", "version": 2 diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/okta_threat_detected_by_okta_threatinsight.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/okta_threat_detected_by_okta_threatinsight.json index 1efcf0474c0499..0101ae04594547 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/okta_threat_detected_by_okta_threatinsight.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/okta_threat_detected_by_okta_threatinsight.json @@ -21,10 +21,11 @@ "severity": "medium", "tags": [ "Elastic", + "Identity", "Okta", + "Continuous Monitoring", "SecOps", - "Monitoring", - "Continuous Monitoring" + "Monitoring" ], "type": "query", "version": 2 diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_administrator_privileges_assigned_to_okta_group.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_administrator_privileges_assigned_to_okta_group.json index 87d5bf3e0f48c4..fad3e3c9224781 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_administrator_privileges_assigned_to_okta_group.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_administrator_privileges_assigned_to_okta_group.json @@ -24,10 +24,11 @@ "severity": "low", "tags": [ "Elastic", + "Identity", "Okta", + "Continuous Monitoring", "SecOps", - "Monitoring", - "Continuous Monitoring" + "Monitoring" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_adobe_hijack_persistence.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_adobe_hijack_persistence.json index a4c62b98fb0609..077147a8ed1a65 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_adobe_hijack_persistence.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_adobe_hijack_persistence.json @@ -17,7 +17,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Persistence" ], "threat": [ { @@ -37,5 +40,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_app_compat_shim.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_app_compat_shim.json index e9e2e044ddc04a..5c467c39f51285 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_app_compat_shim.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_app_compat_shim.json @@ -11,13 +11,16 @@ "language": "eql", "license": "Elastic License", "name": "Installation of Custom Shim Databases", - "query": "/* dependent on wildcard for registry.value */\n\nsequence by process.entity_id with maxspan=5m\n [process where event.type in (\"start\", \"process_started\") and\n not (process.name == \"sdbinst.exe\" and process.parent.name == \"msiexec.exe\")]\n [registry where event.type in (\"creation\", \"change\") and\n wildcard(registry.path, \"HKLM\\\\SOFTWARE\\\\Microsoft\\\\Windows NT\\\\CurrentVersion\\\\AppCompatFlags\\\\Custom\\\\*.sdb\")]\n", + "query": "sequence by process.entity_id with maxspan=5m\n [process where event.type in (\"start\", \"process_started\") and\n not (process.name : \"sdbinst.exe\" and process.parent.name : \"msiexec.exe\")]\n [registry where event.type in (\"creation\", \"change\") and\n wildcard(registry.path, \"HKLM\\\\SOFTWARE\\\\Microsoft\\\\Windows NT\\\\CurrentVersion\\\\AppCompatFlags\\\\Custom\\\\*.sdb\")]\n", "risk_score": 21, "rule_id": "c5ce48a6-7f57-4ee8-9313-3d0024caee10", "severity": "medium", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Persistence" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_attempt_to_create_okta_api_token.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_attempt_to_create_okta_api_token.json index c1d7d51f1401e3..9d1a7c7aef464f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_attempt_to_create_okta_api_token.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_attempt_to_create_okta_api_token.json @@ -24,10 +24,11 @@ "severity": "low", "tags": [ "Elastic", + "Identity", "Okta", + "Continuous Monitoring", "SecOps", - "Monitoring", - "Continuous Monitoring" + "Monitoring" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_attempt_to_deactivate_mfa_for_okta_user_account.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_attempt_to_deactivate_mfa_for_okta_user_account.json index 9cd9572400a6c5..764c60b8294982 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_attempt_to_deactivate_mfa_for_okta_user_account.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_attempt_to_deactivate_mfa_for_okta_user_account.json @@ -24,10 +24,11 @@ "severity": "low", "tags": [ "Elastic", + "Identity", "Okta", + "Continuous Monitoring", "SecOps", - "Identity and Access", - "Continuous Monitoring" + "Identity and Access" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_attempt_to_deactivate_okta_policy.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_attempt_to_deactivate_okta_policy.json index d5c9e505659f6f..9003f6877341fe 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_attempt_to_deactivate_okta_policy.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_attempt_to_deactivate_okta_policy.json @@ -24,10 +24,11 @@ "severity": "low", "tags": [ "Elastic", + "Identity", "Okta", + "Continuous Monitoring", "SecOps", - "Monitoring", - "Continuous Monitoring" + "Monitoring" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_attempt_to_reset_mfa_factors_for_okta_user_account.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_attempt_to_reset_mfa_factors_for_okta_user_account.json index 302618773e323b..4fef3e833a7b6a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_attempt_to_reset_mfa_factors_for_okta_user_account.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_attempt_to_reset_mfa_factors_for_okta_user_account.json @@ -24,10 +24,11 @@ "severity": "low", "tags": [ "Elastic", + "Identity", "Okta", + "Continuous Monitoring", "SecOps", - "Identity and Access", - "Continuous Monitoring" + "Identity and Access" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_azure_automation_account_created.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_azure_automation_account_created.json index 645c025ec47388..5c000967ce44d5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_azure_automation_account_created.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_azure_automation_account_created.json @@ -22,11 +22,12 @@ "rule_id": "df26fd74-1baa-4479-b42e-48da84642330", "severity": "low", "tags": [ + "Elastic", + "Cloud", "Azure", "Continuous Monitoring", - "Elastic", - "Identity and Access", - "SecOps" + "SecOps", + "Identity and Access" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_azure_automation_runbook_created_or_modified.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_azure_automation_runbook_created_or_modified.json index e96700e4090908..28a58643539427 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_azure_automation_runbook_created_or_modified.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_azure_automation_runbook_created_or_modified.json @@ -23,6 +23,7 @@ "severity": "low", "tags": [ "Elastic", + "Cloud", "Azure", "Continuous Monitoring", "SecOps", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_azure_automation_webhook_created.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_azure_automation_webhook_created.json index f31fdcc18978e9..5dde815022283d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_azure_automation_webhook_created.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_azure_automation_webhook_created.json @@ -23,6 +23,7 @@ "severity": "low", "tags": [ "Elastic", + "Cloud", "Azure", "Continuous Monitoring", "SecOps", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_azure_conditional_access_policy_modified.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_azure_conditional_access_policy_modified.json similarity index 99% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_azure_conditional_access_policy_modified.json rename to x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_azure_conditional_access_policy_modified.json index 8d4d4b971316ed..14d6d3d479c6ad 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_azure_conditional_access_policy_modified.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_azure_conditional_access_policy_modified.json @@ -20,6 +20,7 @@ "severity": "medium", "tags": [ "Elastic", + "Cloud", "Azure", "Continuous Monitoring", "SecOps", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_azure_pim_user_added_global_admin.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_azure_pim_user_added_global_admin.json index b8ea2c55dd3f99..24411a40ffc46a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_azure_pim_user_added_global_admin.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_azure_pim_user_added_global_admin.json @@ -22,6 +22,7 @@ "severity": "high", "tags": [ "Elastic", + "Cloud", "Azure", "Continuous Monitoring", "SecOps", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_azure_privileged_identity_management_role_modified.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_azure_privileged_identity_management_role_modified.json similarity index 99% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_azure_privileged_identity_management_role_modified.json rename to x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_azure_privileged_identity_management_role_modified.json index f675a490c4e053..77e955727b2d83 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_azure_privileged_identity_management_role_modified.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_azure_privileged_identity_management_role_modified.json @@ -21,6 +21,7 @@ "severity": "medium", "tags": [ "Elastic", + "Cloud", "Azure", "Continuous Monitoring", "SecOps", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_ec2_network_acl_creation.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_ec2_network_acl_creation.json index 1b98b9744cd5bd..8deaa9924cc1de 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_ec2_network_acl_creation.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_ec2_network_acl_creation.json @@ -27,11 +27,12 @@ "rule_id": "39144f38-5284-4f8e-a2ae-e3fd628d90b0", "severity": "low", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Network", - "Continuous Monitoring" + "Network Security" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_gcp_service_account_created.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_gcp_service_account_created.json index aa9d48459262ed..62e28e588cd0ac 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_gcp_service_account_created.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_gcp_service_account_created.json @@ -22,6 +22,7 @@ "severity": "low", "tags": [ "Elastic", + "Cloud", "GCP", "Continuous Monitoring", "SecOps", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_gpo_schtask_service_creation.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_gpo_schtask_service_creation.json similarity index 96% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_gpo_schtask_service_creation.json rename to x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_gpo_schtask_service_creation.json index fbf6fddcb8c00b..536eda4e214764 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/lateral_movement_gpo_schtask_service_creation.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_gpo_schtask_service_creation.json @@ -17,7 +17,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Persistence" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_iam_group_creation.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_iam_group_creation.json index 0addb86b8d031a..963ac46b7ed942 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_iam_group_creation.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_iam_group_creation.json @@ -25,11 +25,12 @@ "rule_id": "169f3a93-efc7-4df2-94d6-0d9438c310d1", "severity": "low", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Identity and Access", - "Continuous Monitoring" + "Identity and Access" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_kernel_module_activity.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_kernel_module_activity.json index e3dedeef07eb56..b29a8b2384f95c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_kernel_module_activity.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_kernel_module_activity.json @@ -23,7 +23,10 @@ "severity": "low", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection", + "Persistence" ], "threat": [ { @@ -43,5 +46,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_local_scheduled_task_commands.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_local_scheduled_task_commands.json index 78f035318c614a..9e5a6a0eae0412 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_local_scheduled_task_commands.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_local_scheduled_task_commands.json @@ -20,7 +20,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Persistence" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_mfa_disabled_for_azure_user.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_mfa_disabled_for_azure_user.json similarity index 99% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_mfa_disabled_for_azure_user.json rename to x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_mfa_disabled_for_azure_user.json index eda6f5b2bdf62b..8a9f4d4c661e96 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/defense_evasion_mfa_disabled_for_azure_user.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_mfa_disabled_for_azure_user.json @@ -17,6 +17,7 @@ "severity": "medium", "tags": [ "Elastic", + "Cloud", "Azure", "Continuous Monitoring", "SecOps", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_priv_escalation_via_accessibility_features.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_priv_escalation_via_accessibility_features.json index 5899b58bce4d5c..e7f4598a19f33d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_priv_escalation_via_accessibility_features.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_priv_escalation_via_accessibility_features.json @@ -4,7 +4,8 @@ ], "description": "Windows contains accessibility features that may be launched with a key combination before a user has logged in. An adversary can modify the way these programs are launched to get a command prompt or backdoor without logging in to the system.", "index": [ - "winlogbeat-*" + "winlogbeat-*", + "logs-endpoint.events.*" ], "language": "kuery", "license": "Elastic License", @@ -15,7 +16,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Persistence" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_rds_cluster_creation.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_rds_cluster_creation.json index eb77c183d90ea4..06ca022726aad7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_rds_cluster_creation.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_rds_cluster_creation.json @@ -27,11 +27,12 @@ "rule_id": "e14c5fd7-fdd7-49c2-9e5b-ec49d817bc8d", "severity": "low", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Asset Visibility", - "Continuous Monitoring" + "Asset Visibility" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_shell_activity_by_web_server.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_shell_activity_by_web_server.json index 2aaf0012acabf5..ea10fa9bdf865f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_shell_activity_by_web_server.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_shell_activity_by_web_server.json @@ -23,7 +23,10 @@ "severity": "medium", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection", + "Persistence" ], "threat": [ { @@ -43,5 +46,5 @@ } ], "type": "query", - "version": 5 + "version": 6 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_system_shells_via_services.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_system_shells_via_services.json index 32d78480325e62..880101e8d93388 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_system_shells_via_services.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_system_shells_via_services.json @@ -17,7 +17,10 @@ "severity": "medium", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Persistence" ], "threat": [ { @@ -37,5 +40,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_user_account_creation.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_user_account_creation.json index 3f2e00f0976de8..9bc603b5ad5472 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_user_account_creation.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_user_account_creation.json @@ -17,7 +17,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Persistence" ], "threat": [ { @@ -37,5 +40,5 @@ } ], "type": "query", - "version": 4 + "version": 5 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_user_added_as_owner_for_azure_application.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_user_added_as_owner_for_azure_application.json index 8882b87e912918..3fddde78beb336 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_user_added_as_owner_for_azure_application.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_user_added_as_owner_for_azure_application.json @@ -17,6 +17,7 @@ "severity": "low", "tags": [ "Elastic", + "Cloud", "Azure", "Continuous Monitoring", "SecOps", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_user_added_as_owner_for_azure_service_principal.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_user_added_as_owner_for_azure_service_principal.json index f7c0af67692e78..de6482f14d2f0c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_user_added_as_owner_for_azure_service_principal.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_user_added_as_owner_for_azure_service_principal.json @@ -20,6 +20,7 @@ "severity": "low", "tags": [ "Elastic", + "Cloud", "Azure", "Continuous Monitoring", "SecOps", diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_via_application_shimming.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_via_application_shimming.json index f20cc75dfa38b0..c9d56a9c68edb8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_via_application_shimming.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_via_application_shimming.json @@ -4,7 +4,8 @@ ], "description": "The Application Shim was created to allow for backward compatibility of software as the operating system codebase changes over time. This Windows functionality has been abused by attackers to stealthily gain persistence and arbitrary code execution in legitimate Windows processes.", "index": [ - "winlogbeat-*" + "winlogbeat-*", + "logs-endpoint.events.*" ], "language": "kuery", "license": "Elastic License", @@ -15,7 +16,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Persistence" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_via_telemetrycontroller_scheduledtask_hijack.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_via_telemetrycontroller_scheduledtask_hijack.json index b7f4ec5d8a73cc..0622309387f35a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_via_telemetrycontroller_scheduledtask_hijack.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_via_telemetrycontroller_scheduledtask_hijack.json @@ -20,7 +20,10 @@ "severity": "high", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Persistence" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_via_update_orchestrator_service_hijack.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_via_update_orchestrator_service_hijack.json index e512e92a315605..bdddf2eb7e8c7a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_via_update_orchestrator_service_hijack.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/persistence_via_update_orchestrator_service_hijack.json @@ -20,7 +20,10 @@ "severity": "high", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Persistence" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_printspooler_service_suspicious_file.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_printspooler_service_suspicious_file.json index 1a76e077a74656..398e4ca22b757f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_printspooler_service_suspicious_file.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_printspooler_service_suspicious_file.json @@ -21,7 +21,10 @@ "severity": "high", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Privilege Escalation" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_printspooler_suspicious_spl_file.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_printspooler_suspicious_spl_file.json index c5ffe5a9f6a11c..18da4c8075c8f0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_printspooler_suspicious_spl_file.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_printspooler_suspicious_spl_file.json @@ -21,7 +21,10 @@ "severity": "high", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Privilege Escalation" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_root_login_without_mfa.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_root_login_without_mfa.json index 16389d43945f1a..16af96e98448cf 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_root_login_without_mfa.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_root_login_without_mfa.json @@ -24,11 +24,12 @@ "rule_id": "bc0c6f0d-dab0-47a3-b135-0925f0a333bc", "severity": "high", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Identity and Access", - "Continuous Monitoring" + "Identity and Access" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_setgid_bit_set_via_chmod.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_setgid_bit_set_via_chmod.json index e72e58132adeef..ff63d1e38d9505 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_setgid_bit_set_via_chmod.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_setgid_bit_set_via_chmod.json @@ -18,7 +18,10 @@ "severity": "low", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection", + "Privilege Escalation" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_setuid_bit_set_via_chmod.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_setuid_bit_set_via_chmod.json index 5e560097d25455..47490fe08ff129 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_setuid_bit_set_via_chmod.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_setuid_bit_set_via_chmod.json @@ -18,7 +18,10 @@ "severity": "low", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection", + "Privilege Escalation" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_sudoers_file_mod.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_sudoers_file_mod.json index 73a804fcbda8f1..5519f6ce3a9ec2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_sudoers_file_mod.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_sudoers_file_mod.json @@ -17,7 +17,10 @@ "severity": "low", "tags": [ "Elastic", - "Linux" + "Host", + "Linux", + "Threat Detection", + "Privilege Escalation" ], "threat": [ { @@ -37,5 +40,5 @@ } ], "type": "query", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_uac_bypass_diskcleanup_hijack.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_uac_bypass_diskcleanup_hijack.json index b22457db49e491..80b01f90d3cf4b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_uac_bypass_diskcleanup_hijack.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_uac_bypass_diskcleanup_hijack.json @@ -17,7 +17,10 @@ "severity": "medium", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Privilege Escalation" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_uac_bypass_event_viewer.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_uac_bypass_event_viewer.json index 740ff47e5abe5d..415111c7258286 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_uac_bypass_event_viewer.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_uac_bypass_event_viewer.json @@ -17,7 +17,10 @@ "severity": "low", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Privilege Escalation" ], "threat": [ { @@ -37,5 +40,5 @@ } ], "type": "query", - "version": 3 + "version": 4 } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_uac_sdclt.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_uac_sdclt.json new file mode 100644 index 00000000000000..ce82db3a96e111 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_uac_sdclt.json @@ -0,0 +1,44 @@ +{ + "author": [ + "Elastic" + ], + "description": "Identifies User Account Control (UAC) bypass via sdclt.exe. Attackers bypass UAC to stealthily execute code with elevated permissions.", + "from": "now-9m", + "index": [ + "logs-endpoint.events.*", + "winlogbeat-*" + ], + "language": "eql", + "license": "Elastic License", + "name": "Bypass UAC via Sdclt", + "query": "/* add winlogbeat-* when process.code_signature.* fields are populated */\n\nsequence with maxspan=1m\n [process where event.type in (\"start\", \"process_started\") and process.name : \"sdclt.exe\" and\n /* uncomment once in winlogbeat */\n /* process.code_signature.subject_name == \"Microsoft Corporation\" and process.code_signature.trusted == true and */\n process.args : \"/kickoffelev\"\n ] by process.entity_id\n [process where event.type in (\"start\", \"process_started\") and process.parent.name : \"sdclt.exe\" and\n not (process.executable : \"C:\\\\Windows\\\\System32\\\\sdclt.exe\" or\n process.executable : \"C:\\\\Windows\\\\System32\\\\control.exe\" or\n process.executable : \"C:\\\\Windows\\\\SysWOW64\\\\sdclt.exe\" or\n process.executable : \"C:\\\\Windows\\\\SysWOW64\\\\control.exe\")\n ] by process.parent.entity_id\n", + "risk_score": 21, + "rule_id": "9b54e002-034a-47ac-9307-ad12c03fa900", + "severity": "high", + "tags": [ + "Elastic", + "Host", + "Windows", + "Threat Detection", + "Privilege Escalation" + ], + "threat": [ + { + "framework": "MITRE ATT&CK", + "tactic": { + "id": "TA0004", + "name": "Privilege Escalation", + "reference": "https://attack.mitre.org/tactics/TA0004/" + }, + "technique": [ + { + "id": "T1088", + "name": "Bypass User Account Control", + "reference": "https://attack.mitre.org/techniques/T1088/" + } + ] + } + ], + "type": "eql", + "version": 1 +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_unusual_parentchild_relationship.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_unusual_parentchild_relationship.json index 0cac8561c7e9cd..ad871716a67aab 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_unusual_parentchild_relationship.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_unusual_parentchild_relationship.json @@ -17,7 +17,10 @@ "severity": "medium", "tags": [ "Elastic", - "Windows" + "Host", + "Windows", + "Threat Detection", + "Privilege Escalation" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_updateassumerolepolicy.json b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_updateassumerolepolicy.json index 55947e00170ae7..3d1e66497bd2e9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_updateassumerolepolicy.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules/privilege_escalation_updateassumerolepolicy.json @@ -24,11 +24,12 @@ "rule_id": "a60326d7-dca7-4fb7-93eb-1ca03a1febbd", "severity": "low", "tags": [ - "AWS", "Elastic", + "Cloud", + "AWS", + "Continuous Monitoring", "SecOps", - "Identity and Access", - "Continuous Monitoring" + "Identity and Access" ], "threat": [ { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts index 8af622e6a128b9..fb4763a982f43f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts @@ -105,7 +105,7 @@ export interface RuleAlertType extends Alert { } // eslint-disable-next-line @typescript-eslint/no-explicit-any -export interface IRuleStatusAttributes extends Record { +export interface IRuleStatusSOAttributes extends Record { alertId: string; // created alert id. statusDate: StatusDate; lastFailureAt: LastFailureAt | null | undefined; @@ -119,21 +119,35 @@ export interface IRuleStatusAttributes extends Record { searchAfterTimeDurations: string[] | null | undefined; } +export interface IRuleStatusResponseAttributes { + alert_id: string; // created alert id. + status_date: StatusDate; + last_failure_at: LastFailureAt | null | undefined; + last_failure_message: LastFailureMessage | null | undefined; + last_success_at: LastSuccessAt | null | undefined; + last_success_message: LastSuccessMessage | null | undefined; + status: JobStatus | null | undefined; + last_look_back_date: string | null | undefined; + gap: string | null | undefined; + bulk_create_time_durations: string[] | null | undefined; + search_after_time_durations: string[] | null | undefined; +} + export interface RuleStatusResponse { [key: string]: { - current_status: IRuleStatusAttributes | null | undefined; - failures: IRuleStatusAttributes[] | null | undefined; + current_status: IRuleStatusResponseAttributes | null | undefined; + failures: IRuleStatusResponseAttributes[] | null | undefined; }; } export interface IRuleSavedAttributesSavedObjectAttributes - extends IRuleStatusAttributes, + extends IRuleStatusSOAttributes, SavedObjectAttributes {} export interface IRuleStatusSavedObject { type: string; id: string; - attributes: Array>; + attributes: Array>; references: unknown[]; updated_at: string; version: string; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts index cbf70f3119b315..4559a658c9583c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts @@ -19,7 +19,7 @@ import { } from '../../../../../../../../src/core/server'; import { loggingSystemMock } from '../../../../../../../../src/core/server/mocks'; import { RuleTypeParams } from '../../types'; -import { IRuleStatusAttributes } from '../../rules/types'; +import { IRuleStatusSOAttributes } from '../../rules/types'; import { ruleStatusSavedObjectType } from '../../rules/saved_object_mappings'; import { getListArrayMock } from '../../../../../common/detection_engine/schemas/types/lists.mock'; import { RulesSchema } from '../../../../../common/detection_engine/schemas/response'; @@ -555,7 +555,7 @@ export const sampleDocSearchResultsWithSortId = ( export const sampleRuleGuid = '04128c15-0d1b-4716-a4c5-46997ac7f3bd'; export const sampleIdGuid = 'e1e08ddc-5e37-49ff-a258-5393aa44435a'; -export const exampleRuleStatus: () => SavedObject = () => ({ +export const exampleRuleStatus: () => SavedObject = () => ({ type: ruleStatusSavedObjectType, id: '042e6d90-7069-11ea-af8b-0f8ae4fa817e', attributes: { @@ -577,8 +577,10 @@ export const exampleRuleStatus: () => SavedObject = () => }); export const exampleFindRuleStatusResponse: ( - mockStatuses: Array> -) => SavedObjectsFindResponse = (mockStatuses = [exampleRuleStatus()]) => ({ + mockStatuses: Array> +) => SavedObjectsFindResponse = ( + mockStatuses = [exampleRuleStatus()] +) => ({ total: 1, per_page: 6, page: 1, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_or_create_rule_statuses.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_or_create_rule_statuses.ts index 913efbe04aa16f..1ddec9cd151483 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_or_create_rule_statuses.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_or_create_rule_statuses.ts @@ -6,7 +6,7 @@ import { SavedObject } from 'src/core/server'; -import { IRuleStatusAttributes } from '../rules/types'; +import { IRuleStatusSOAttributes } from '../rules/types'; import { RuleStatusSavedObjectsClient } from './rule_status_saved_objects_client'; import { getRuleStatusSavedObjects } from './get_rule_status_saved_objects'; @@ -18,7 +18,7 @@ interface RuleStatusParams { export const createNewRuleStatus = async ({ alertId, ruleStatusClient, -}: RuleStatusParams): Promise> => { +}: RuleStatusParams): Promise> => { const now = new Date().toISOString(); return ruleStatusClient.create({ alertId, @@ -38,7 +38,7 @@ export const createNewRuleStatus = async ({ export const getOrCreateRuleStatuses = async ({ alertId, ruleStatusClient, -}: RuleStatusParams): Promise>> => { +}: RuleStatusParams): Promise>> => { const ruleStatuses = await getRuleStatusSavedObjects({ alertId, ruleStatusClient, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_rule_status_saved_objects.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_rule_status_saved_objects.ts index 828b4ea41096ed..72a271fb2606fe 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_rule_status_saved_objects.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_rule_status_saved_objects.ts @@ -5,7 +5,7 @@ */ import { SavedObjectsFindResponse } from 'kibana/server'; -import { IRuleStatusAttributes } from '../rules/types'; +import { IRuleStatusSOAttributes } from '../rules/types'; import { MAX_RULE_STATUSES } from './rule_status_service'; import { RuleStatusSavedObjectsClient } from './rule_status_saved_objects_client'; @@ -17,7 +17,7 @@ interface GetRuleStatusSavedObject { export const getRuleStatusSavedObjects = async ({ alertId, ruleStatusClient, -}: GetRuleStatusSavedObject): Promise> => { +}: GetRuleStatusSavedObject): Promise> => { return ruleStatusClient.find({ perPage: MAX_RULE_STATUSES, sortField: 'statusDate', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/rule_status_saved_objects_client.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/rule_status_saved_objects_client.ts index 4b5faeb5b9d274..f6a08852ac8d55 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/rule_status_saved_objects_client.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/rule_status_saved_objects_client.ts @@ -12,17 +12,17 @@ import { SavedObjectsFindResponse, } from '../../../../../../../src/core/server'; import { ruleStatusSavedObjectType } from '../rules/saved_object_mappings'; -import { IRuleStatusAttributes } from '../rules/types'; +import { IRuleStatusSOAttributes } from '../rules/types'; export interface RuleStatusSavedObjectsClient { find: ( options?: Omit - ) => Promise>; - create: (attributes: IRuleStatusAttributes) => Promise>; + ) => Promise>; + create: (attributes: IRuleStatusSOAttributes) => Promise>; update: ( id: string, - attributes: Partial - ) => Promise>; + attributes: Partial + ) => Promise>; delete: (id: string) => Promise<{}>; } @@ -30,7 +30,10 @@ export const ruleStatusSavedObjectsClientFactory = ( savedObjectsClient: SavedObjectsClientContract ): RuleStatusSavedObjectsClient => ({ find: (options) => - savedObjectsClient.find({ ...options, type: ruleStatusSavedObjectType }), + savedObjectsClient.find({ + ...options, + type: ruleStatusSavedObjectType, + }), create: (attributes) => savedObjectsClient.create(ruleStatusSavedObjectType, attributes), update: (id, attributes) => savedObjectsClient.update(ruleStatusSavedObjectType, id, attributes), delete: (id) => savedObjectsClient.delete(ruleStatusSavedObjectType, id), diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/rule_status_service.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/rule_status_service.ts index 8fdbe282eece5d..433ad4e2affea4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/rule_status_service.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/rule_status_service.ts @@ -6,7 +6,7 @@ import { assertUnreachable } from '../../../../common/utility_types'; import { JobStatus } from '../../../../common/detection_engine/schemas/common/schemas'; -import { IRuleStatusAttributes } from '../rules/types'; +import { IRuleStatusSOAttributes } from '../rules/types'; import { getOrCreateRuleStatuses } from './get_or_create_rule_statuses'; import { RuleStatusSavedObjectsClient } from './rule_status_saved_objects_client'; @@ -30,9 +30,9 @@ export const buildRuleStatusAttributes: ( status: JobStatus, message?: string, attributes?: Attributes -) => Partial = (status, message, attributes = {}) => { +) => Partial = (status, message, attributes = {}) => { const now = new Date().toISOString(); - const baseAttributes: Partial = { + const baseAttributes: Partial = { ...attributes, status, statusDate: now, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index 838ac2558b0389..bb3a0b4fa6f08c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -8,7 +8,6 @@ import { Logger, KibanaRequest } from 'src/core/server'; -import { get } from 'lodash'; import { SIGNALS_ID, DEFAULT_SEARCH_AFTER_PAGE_SIZE, @@ -62,6 +61,8 @@ import { buildEqlSearchRequest } from '../../../../common/detection_engine/get_q import { bulkInsertSignals } from './single_bulk_create'; import { buildSignalFromEvent, buildSignalGroupFromSequence } from './build_bulk_body'; import { createThreatSignals } from './threat_mapping/create_threat_signals'; +import { getIndexVersion } from '../routes/index/get_index_version'; +import { MIN_EQL_RULE_INDEX_VERSION } from '../routes/index/get_signals_template'; export const signalRulesAlertType = ({ logger, @@ -119,17 +120,6 @@ export const signalRulesAlertType = ({ type, exceptionsList, } = params; - const outputIndexTemplateMapping: unknown = await services.callCluster( - 'indices.getTemplate', - { name: outputIndex } - ); - const signalMappingVersion: number | undefined = get(outputIndexTemplateMapping, [ - outputIndex, - 'version', - ]); - if (signalMappingVersion !== undefined && typeof signalMappingVersion !== 'number') { - throw new Error('Found non-numeric value for "version" in output index template'); - } const searchAfterSize = Math.min(maxSignals, DEFAULT_SEARCH_AFTER_PAGE_SIZE); let hasError: boolean = false; @@ -457,14 +447,24 @@ export const signalRulesAlertType = ({ if (query === undefined) { throw new Error('EQL query rule must have a query defined'); } - const MIN_EQL_RULE_TEMPLATE_VERSION = 2; - if ( - signalMappingVersion === undefined || - signalMappingVersion < MIN_EQL_RULE_TEMPLATE_VERSION - ) { - throw new Error( - `EQL based rules require an update to version ${MIN_EQL_RULE_TEMPLATE_VERSION} of the detection alerts index mapping` - ); + try { + const signalIndexVersion = await getIndexVersion(services.callCluster, outputIndex); + if ( + signalIndexVersion === undefined || + signalIndexVersion < MIN_EQL_RULE_INDEX_VERSION + ) { + throw new Error( + `EQL based rules require an update to version ${MIN_EQL_RULE_INDEX_VERSION} of the detection alerts index mapping` + ); + } + } catch (err) { + if (err.statusCode === 403) { + throw new Error( + `EQL based rules require the user that created it to have the view_index_metadata, read, and write permissions for index: ${outputIndex}` + ); + } else { + throw err; + } } const inputIndex = await getInputIndex(services, version, index); const request = buildEqlSearchRequest( diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.test.ts index 7b7c40f0c43550..c4869f024a9777 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.test.ts @@ -105,7 +105,9 @@ describe('singleSearchAfter', () => { timestampOverride: undefined, buildRuleMessage, }); - expect(searchErrors).toEqual(['reason: some reason, type: some type, caused by: some reason']); + expect(searchErrors).toEqual([ + 'reason: "some reason" type: "some type" caused by reason: "some reason" caused by type: "some type"', + ]); }); test('if singleSearchAfter works with a given sort id', async () => { const searchAfterSortId = '1234567891111'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts index 157f741439bd23..894e934ff0247a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts @@ -878,7 +878,7 @@ describe('utils', () => { ]; const createdErrors = createErrorsFromShard({ errors }); expect(createdErrors).toEqual([ - 'reason: some reason, type: some type, caused by: some reason', + 'reason: "some reason" type: "some type" caused by reason: "some reason" caused by type: "some type"', ]); }); @@ -917,8 +917,54 @@ describe('utils', () => { ]; const createdErrors = createErrorsFromShard({ errors }); expect(createdErrors).toEqual([ - 'reason: some reason, type: some type, caused by: some reason', - 'reason: some reason 2, type: some type 2, caused by: some reason 2', + 'reason: "some reason" type: "some type" caused by reason: "some reason" caused by type: "some type"', + 'reason: "some reason 2" type: "some type 2" caused by reason: "some reason 2" caused by type: "some type 2"', + ]); + }); + + test('You can have missing values for the shard errors and get the expected output of an empty string', () => { + const errors: ShardError[] = [ + { + shard: 1, + index: 'index-123', + node: 'node-123', + reason: {}, + }, + ]; + const createdErrors = createErrorsFromShard({ errors }); + expect(createdErrors).toEqual(['']); + }); + + test('You can have a single value for the shard errors and get expected output without extra spaces anywhere', () => { + const errors: ShardError[] = [ + { + shard: 1, + index: 'index-123', + node: 'node-123', + reason: { + reason: 'some reason something went wrong', + }, + }, + ]; + const createdErrors = createErrorsFromShard({ errors }); + expect(createdErrors).toEqual(['reason: "some reason something went wrong"']); + }); + + test('You can have two values for the shard errors and get expected output with one space exactly between the two values', () => { + const errors: ShardError[] = [ + { + shard: 1, + index: 'index-123', + node: 'node-123', + reason: { + reason: 'some reason something went wrong', + caused_by: { type: 'some type' }, + }, + }, + ]; + const createdErrors = createErrorsFromShard({ errors }); + expect(createdErrors).toEqual([ + 'reason: "some reason something went wrong" caused by type: "some type"', ]); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts index ac10f5ed9a72d8..e2b39b8d0a8c83 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts @@ -511,7 +511,23 @@ export const getSignalTimeTuples = ({ */ export const createErrorsFromShard = ({ errors }: { errors: ShardError[] }): string[] => { return errors.map((error) => { - return `reason: ${error.reason.reason}, type: ${error.reason.caused_by.type}, caused by: ${error.reason.caused_by.reason}`; + const { + reason: { + reason, + type, + caused_by: { reason: causedByReason, type: causedByType } = { + reason: undefined, + type: undefined, + }, + } = {}, + } = error; + + return [ + ...(reason != null ? [`reason: "${reason}"`] : []), + ...(type != null ? [`type: "${type}"`] : []), + ...(causedByReason != null ? [`caused by reason: "${causedByReason}"`] : []), + ...(causedByType != null ? [`caused by type: "${causedByType}"`] : []), + ].join(' '); }); }; diff --git a/x-pack/plugins/security_solution/server/lib/framework/kibana_framework_adapter.ts b/x-pack/plugins/security_solution/server/lib/framework/kibana_framework_adapter.ts index 6d9e9b13bc356a..e36fb1144e93fc 100644 --- a/x-pack/plugins/security_solution/server/lib/framework/kibana_framework_adapter.ts +++ b/x-pack/plugins/security_solution/server/lib/framework/kibana_framework_adapter.ts @@ -149,14 +149,7 @@ export class KibanaBackendFrameworkAdapter implements FrameworkAdapter { } public getIndexPatternsService(request: FrameworkRequest): FrameworkIndexPatternsService { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const callCluster = async (endpoint: string, params?: Record) => - this.callWithRequest(request, endpoint, { - ...params, - allowNoIndices: true, - }); - - return new IndexPatternsFetcher(callCluster); + return new IndexPatternsFetcher(request.context.core.elasticsearch.client.asCurrentUser, true); } } diff --git a/x-pack/plugins/security_solution/server/lib/types.ts b/x-pack/plugins/security_solution/server/lib/types.ts index 29db38bbbea681..c735412aedbf53 100644 --- a/x-pack/plugins/security_solution/server/lib/types.ts +++ b/x-pack/plugins/security_solution/server/lib/types.ts @@ -48,21 +48,26 @@ export interface ShardsResponse { failures?: ShardError[]; } -export interface ShardError { +/** + * This type is being very conservative with the partials to not expect anything to + * be guaranteed on the type as we don't have regular and proper types of ShardError. + * Once we do, remove this type for the regular ShardError type from the elastic library. + */ +export type ShardError = Partial<{ shard: number; index: string; node: string; - reason: { + reason: Partial<{ type: string; reason: string; index_uuid: string; index: string; - caused_by: { + caused_by: Partial<{ type: string; reason: string; - }; - }; -} + }>; + }>; +}>; export interface SearchResponse { took: number; diff --git a/x-pack/plugins/security_solution/server/search_strategy/index_fields/index.ts b/x-pack/plugins/security_solution/server/search_strategy/index_fields/index.ts index bc461f3885a70b..aef092949a47e5 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/index_fields/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/index_fields/index.ts @@ -30,9 +30,7 @@ export const securitySolutionIndexFieldsProvider = (): ISearchStrategy< from( new Promise(async (resolve) => { const { elasticsearch } = context.core; - const indexPatternsFetcher = new IndexPatternsFetcher( - elasticsearch.legacy.client.callAsCurrentUser - ); + const indexPatternsFetcher = new IndexPatternsFetcher(elasticsearch.client.asCurrentUser); const dedupeIndices = dedupeIndexName(request.indices); const responsesIndexFields = await Promise.all( diff --git a/x-pack/plugins/task_manager/server/MONITORING.md b/x-pack/plugins/task_manager/server/MONITORING.md new file mode 100644 index 00000000000000..4960086411e9a1 --- /dev/null +++ b/x-pack/plugins/task_manager/server/MONITORING.md @@ -0,0 +1,265 @@ +# Task Manager Monitoring + +Task Manager has an internal monitoring mechanism in which keeps track of a variety of metrics which are exposed via a `health` api endpoint and Kibana Server Log debug messaging. + +## Exposed Metrics +There are three different sections to the stats returned by the `health` api. +- `configuration`: Summarizes Task Manager's current configuration. +- `workload`: Summarizes the workload in the current deployment. +- `runtime`: Tracks Task Manager's performance. + +### Configuring the Stats +There are four new configurations: + +- `xpack.task_manager.monitored_stats_required_freshness` - The _required freshness_ of critical "Hot" stats, which means that if key stats (last polling cycle time, for example) haven't been refreshed within the specified duration, the `_health` endpoint and service will report an `Error` status. By default this is inferred from the configured `poll_interval` and is set to `poll_interval` plus a `1s` buffer. +- `xpack.task_manager.monitored_aggregated_stats_refresh_rate` - Dictates how often we refresh the "Cold" metrics. These metrics require an aggregation against Elasticsearch and add load to the system, hence we want to limit how often we execute these. We also inffer the _required freshness_ of these "Cold" metrics from this configuration, which means that if these stats have not been updated within the required duration then the `_health` endpoint and service will report an `Error` status. This covers the entire `workload` section of the stats. By default this is configured to `60s`, and as a result the _required freshness_ defaults to `61s` (refresh plus a `1s` buffer). +- `xpack.task_manager.monitored_stats_running_average_window`- Dictates the size of the window used to calculate the running average of various "Hot" stats, such as the time it takes to run a task, the _drift_ that tasks experience etc. These stats are collected throughout the lifecycle of tasks and this window will dictate how large the queue we keep in memory would be, and how many values we need to calculate the average against. We do not calculate the average on *every* new value, but rather only when the time comes to summarize the stats before logging them or returning them to the API endpoint. +- `xpack.task_manager.monitored_task_execution_thresholds`- Configures the threshold of failed task executions at which point the `warn` or `error` health status will be set either at a default level or a custom level for specific task types. This will allow you to mark the health as `error` when any task type failes 90% of the time, but set it to `error` at 50% of the time for task types that you consider critical. This value can be set to any number between 0 to 100, and a threshold is hit when the value *exceeds* this number. This means that you can avoid setting the status to `error` by setting the threshold at 100, or hit `error` the moment any task failes by setting the threshold to 0 (as it will exceed 0 once a single failer occurs). + +For example, in your `Kibana.yml`: +``` +xpack.task_manager.monitored_stats_required_freshness: 5000 +xpack.task_manager.monitored_aggregated_stats_refresh_rate: 60000 +xpack.task_manager.monitored_stats_running_average_window: 50 +xpack.task_manager.monitored_task_execution_thresholds: + default: + error_threshold: 70 + warn_threshold: 50 + custom: + "alerting:always-firing": + error_threshold: 50 + warn_threshold: 0 +``` + +## Consuming Health Stats +Task Manager exposes a `/api/task_manager/_health` api which returns the _latest_ stats. +Calling this API is designed to be fast and doesn't actually perform any checks- rather it returns the result of the latest stats in the system, and is design in such a way that you could call it from an external service on a regular basis without worrying that you'll be adding substantial load to the system. + +Additionally, the metrics are logged out into Task Manager's `DEBUG` logger at a regular cadence (dictated by the Polling Interval). +If you wish to enable DEBUG logging in your Kibana instance, you will need to add the following to your `Kibana.yml`: +``` +logging: + loggers: + - context: plugins.taskManager + appenders: [console] + level: debug +``` + +Please bear in mind that these stats are logged as often as your `poll_interval` configuration, which means it could add substantial noise to your logs. +We would recommend only enabling this level of logging temporarily. + +### Understanding the Exposed Stats + +As mentioned above, the `health` api exposes three sections: `configuration`, `workload` and `runtime`. +Each section has a `timestamp` and a `status` which indicates when the last update to this setion took place and whether the health of this section was evaluated as `OK`, `Warning` or `Error`. + +The root has its own `status` which indicate the state of the system overall as infered from the `status` of the section. +An `Error` status in any section will cause the whole system to display as `Error`. +A `Warning` status in any section will cause the whole system to display as `Warning`. +An `OK` status will only be displayed when all sections are marked as `OK`. + +The root `timestamp` is the time in which the summary was exposed (either to the DEBUG logger or the http api) and the `last_update` is the last time any one of the sections was updated. + +#### The Configuration Section +The `configuration` section summarizes Task Manager's current configuration, including dynamic configurations which change over time, such as `poll_interval` and `max_workers` which adjust in reaction to changing load on the system. + +These are "Hot" stats which are updated whenever a change happens in the configuration. + +#### The Workload Section +The `workload` which summarizes the work load in the current deployment, listing the tasks in the system, their types and what their current status is. + +It includes three sub sections: + - The number of tasks scheduled in the system, broken down by type and status. + - The number of idle `overdue` tasks, whose `runAt` has expired. + - Execution density in the next minute or so (configurable), which shows how many tasks are scheduled to execute in the scope of each polling interval. This can give us an idea of how much load there is on the current Kibana deployment. + +These are "Cold" stat which are updated at a regular cadence, configured by the `monitored_aggregated_stats_refresh_rate` config. + +#### The Runtime Section +The `runtime` tracks Task Manager's performance as it runs, making note of task execution time, _drift_ etc. +These include: + - The time it takes a task to run (mean and median, using a configurable running average window, `50` by default) + - The average _drift_ that tasks experience (mean and median, using the same configurable running average window as above). Drift tells us how long after a task's scheduled a task typically executes. + - The polling rate (the timestamp of the last time a polling cycle completed) and the result [`No tasks | Filled task pool | Unexpectedly ran out of workers`] frequency the past 50 polling cycles (using the same window size as the one used for running averages) + - The `Success | Retry | Failure ratio` by task type. This is different than the workload stats which tell you what's in the queue, but ca't keep track of retries and of non recurring tasks as they're wiped off the index when completed. + +These are "Hot" stats which are updated reactively as Tasks are executed and interacted with. + +### Example Stats + +For example, if you _curl_ the `/api/task_manager/_health` endpoint, you might get these stats: +``` +{ + /* the time these stats were returned by the api */ + "timestamp": "2020-10-05T18:26:11.346Z", + /* the overall status of the system */ + "status": "OK", + /* last time any stat was updated in this output */ + "last_update": "2020-10-05T17:57:55.411Z", + "stats": { + "configuration": { /* current configuration of TM */ + "timestamp": "2020-10-05T17:56:06.507Z", + "status": "OK", + "value": { + "max_workers": 10, + "poll_interval": 3000, + "request_capacity": 1000, + "max_poll_inactivity_cycles": 10, + "monitored_aggregated_stats_refresh_rate": 60000, + "monitored_stats_running_average_window": 50 + } + }, + "workload": { /* The workload of this deployment */ + "timestamp": "2020-10-05T17:57:06.534Z", + "status": "OK", + "value": { + "count": 6, /* count of tasks in the system */ + "task_types": { /* what tasks are there and what status are they in */ + "actions_telemetry": { + "count": 1, + "status": { + "idle": 1 + } + }, + "alerting_telemetry": { + "count": 1, + "status": { + "idle": 1 + } + }, + "apm-telemetry-task": { + "count": 1, + "status": { + "idle": 1 + } + }, + "endpoint:user-artifact-packager": { + "count": 1, + "status": { + "idle": 1 + } + }, + "lens_telemetry": { + "count": 1, + "status": { + "idle": 1 + } + }, + "session_cleanup": { + "count": 1, + "status": { + "idle": 1 + } + } + }, + + /* Frequency of recurring tasks schedules */ + "schedule": [ + ["60s", 1], /* 1 task, every 60s */ + ["3600s", 3], /* 3 tasks every hour */ + ["720m", 1] + ], + /* There are no overdue tasks in this system at the moment */ + "overdue": 0, + /* This is the schedule density, it shows a histogram of all the polling intervals in the next minute (or, if + pollInterval is configured unusually high it will show a min of 2 refresh intervals into the future, and a max of 50 buckets). + Here we see that on the 3rd polling interval from *now* (which is ~9 seconds from now, as pollInterval is `3s`) there is one task due to run. + We also see that there are 5 due two intervals later, which is fine as we have a max workers of `10` + */ + "estimated_schedule_density": [0, 0, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + } + }, + "runtime": { + "timestamp": "2020-10-05T17:57:55.411Z", + "status": "OK", + "value": { + "polling": { + /* When was the last polling cycle? */ + "last_successful_poll": "2020-10-05T17:57:55.411Z", + /* What is the frequency of polling cycle result? + Here we see 94% of "NoTasksClaimed" and 6% "PoolFilled" */ + "result_frequency_percent_as_number": { + "NoTasksClaimed": 94, + "RanOutOfCapacity": 0, /* This is a legacy result, we might want to rename - it tells us when a polling cycle resulted in claiming more tasks than we had workers for, butt he name doesn't make much sense outside of the context of the code */ + "PoolFilled": 6 + } + }, + /* on average, the tasks in this deployment run 1.7s after their scheduled time */ + "drift": { + "mean": 1720, + "median": 2276 + }, + "execution": { + "duration": { + /* on average, the `endpoint:user-artifact-packager` tasks take 15ms to run */ + "endpoint:user-artifact-packager": { + "mean": 15, + "median": 14.5 + }, + "session_cleanup": { + "mean": 28, + "median": 28 + }, + "lens_telemetry": { + "mean": 100, + "median": 100 + }, + "actions_telemetry": { + "mean": 135, + "median": 135 + }, + "alerting_telemetry": { + "mean": 197, + "median": 197 + }, + "apm-telemetry-task": { + "mean": 1347, + "median": 1347 + } + }, + "result_frequency_percent_as_number": { + /* and 100% of `endpoint:user-artifact-packager` have completed in success (within the running average window, so the past 50 runs (by default, configrable by `monitored_stats_running_average_window`) */ + "endpoint:user-artifact-packager": { + "status": "OK", + "Success": 100, + "RetryScheduled": 0, + "Failed": 0 + }, + "session_cleanup": { + /* `error` status as 90% of results are `Failed` */ + "status": "error", + "Success": 5, + "RetryScheduled": 5, + "Failed": 90 + }, + "lens_telemetry": { + "status": "OK", + "Success": 100, + "RetryScheduled": 0, + "Failed": 0 + }, + "actions_telemetry": { + "status": "OK", + "Success": 100, + "RetryScheduled": 0, + "Failed": 0 + }, + "alerting_telemetry": { + "status": "OK", + "Success": 100, + "RetryScheduled": 0, + "Failed": 0 + }, + "apm-telemetry-task": { + "status": "OK", + "Success": 100, + "RetryScheduled": 0, + "Failed": 0 + } + } + } + } + } + } +} +``` diff --git a/x-pack/plugins/task_manager/server/README.md b/x-pack/plugins/task_manager/server/README.md index fd2409a7db0a53..a0b35ad094537c 100644 --- a/x-pack/plugins/task_manager/server/README.md +++ b/x-pack/plugins/task_manager/server/README.md @@ -48,6 +48,10 @@ The task_manager can be configured via `taskManager` config options (e.g. `taskM - `override_num_workers`: An object of `taskType: number` that overrides the `num_workers` for tasks - For example: `task_manager.override_num_workers.reporting: 2` would override the number of workers occupied by tasks of type `reporting` - This allows sysadmins to tweak the operational performance of Kibana, allowing more or fewer tasks of a specific type to run simultaneously +- `monitored_aggregated_stats_refresh_rate` - Dictates how often we refresh the "Cold" metrics. Learn More: [./MONITORING](./MONITORING.MD) +- `monitored_stats_running_average_window`- Dictates the size of the window used to calculate the running average of various "Hot" stats. Learn More: [./MONITORING](./MONITORING.MD) +- `monitored_stats_required_freshness` - Dictates the _required freshness_ of critical "Hot" stats. Learn More: [./MONITORING](./MONITORING.MD) +- `monitored_task_execution_thresholds`- Dictates the threshold of failed task executions. Learn More: [./MONITORING](./MONITORING.MD) ## Task definitions @@ -460,3 +464,9 @@ The task manager's public API is create / delete / list. Updates aren't directly node scripts/functional_tests_server.js --config x-pack/test/plugin_api_integration/config.ts node scripts/functional_test_runner --config x-pack/test/plugin_api_integration/config.ts ``` + +## Monitoring + +Task Manager exposes runtime statistics which enable basic observability into its inner workings and makes it possible to monitor the system from external services. + +Learn More: [./MONITORING](./MONITORING.MD) \ No newline at end of file diff --git a/x-pack/plugins/task_manager/server/config.test.ts b/x-pack/plugins/task_manager/server/config.test.ts index d5bbbe65582f18..d2d5ac8f22a1fd 100644 --- a/x-pack/plugins/task_manager/server/config.test.ts +++ b/x-pack/plugins/task_manager/server/config.test.ts @@ -15,6 +15,16 @@ describe('config validation', () => { "max_attempts": 3, "max_poll_inactivity_cycles": 10, "max_workers": 10, + "monitored_aggregated_stats_refresh_rate": 60000, + "monitored_stats_required_freshness": 4000, + "monitored_stats_running_average_window": 50, + "monitored_task_execution_thresholds": Object { + "custom": Object {}, + "default": Object { + "error_threshold": 90, + "warn_threshold": 80, + }, + }, "poll_interval": 3000, "request_capacity": 1000, } @@ -31,4 +41,147 @@ describe('config validation', () => { `"[index]: \\".tasks\\" is an invalid Kibana Task Manager index, as it is already in use by the ElasticSearch Tasks Manager"` ); }); + + test('the required freshness of the monitored stats config must always be less-than-equal to the poll interval', () => { + const config: Record = { + monitored_stats_required_freshness: 100, + }; + expect(() => { + configSchema.validate(config); + }).toThrowErrorMatchingInlineSnapshot( + `"The specified monitored_stats_required_freshness (100) is invalid, as it is below the poll_interval (3000)"` + ); + }); + + test('the default required freshness of the monitored stats is poll interval with a slight buffer', () => { + const config: Record = {}; + expect(configSchema.validate(config)).toMatchInlineSnapshot(` + Object { + "enabled": true, + "index": ".kibana_task_manager", + "max_attempts": 3, + "max_poll_inactivity_cycles": 10, + "max_workers": 10, + "monitored_aggregated_stats_refresh_rate": 60000, + "monitored_stats_required_freshness": 4000, + "monitored_stats_running_average_window": 50, + "monitored_task_execution_thresholds": Object { + "custom": Object {}, + "default": Object { + "error_threshold": 90, + "warn_threshold": 80, + }, + }, + "poll_interval": 3000, + "request_capacity": 1000, + } + `); + }); + + test('the custom monitored_task_execution_thresholds can be configured at task type', () => { + const config: Record = { + monitored_task_execution_thresholds: { + custom: { + 'alerting:always-fires': { + error_threshold: 50, + warn_threshold: 30, + }, + }, + }, + }; + expect(configSchema.validate(config)).toMatchInlineSnapshot(` + Object { + "enabled": true, + "index": ".kibana_task_manager", + "max_attempts": 3, + "max_poll_inactivity_cycles": 10, + "max_workers": 10, + "monitored_aggregated_stats_refresh_rate": 60000, + "monitored_stats_required_freshness": 4000, + "monitored_stats_running_average_window": 50, + "monitored_task_execution_thresholds": Object { + "custom": Object { + "alerting:always-fires": Object { + "error_threshold": 50, + "warn_threshold": 30, + }, + }, + "default": Object { + "error_threshold": 90, + "warn_threshold": 80, + }, + }, + "poll_interval": 3000, + "request_capacity": 1000, + } + `); + }); + + test('the monitored_task_execution_thresholds ensures that the default warn_threshold is lt the default error_threshold', () => { + const config: Record = { + monitored_task_execution_thresholds: { + default: { + warn_threshold: 80, + error_threshold: 70, + }, + }, + }; + expect(() => { + configSchema.validate(config); + }).toThrowErrorMatchingInlineSnapshot( + `"[monitored_task_execution_thresholds.default]: warn_threshold (80) must be less than, or equal to, error_threshold (70)"` + ); + }); + + test('the monitored_task_execution_thresholds allows the default warn_threshold to equal the default error_threshold', () => { + const config: Record = { + monitored_task_execution_thresholds: { + default: { + warn_threshold: 70, + error_threshold: 70, + }, + }, + }; + expect(() => { + configSchema.validate(config); + }).not.toThrowError(); + }); + + test('the monitored_task_execution_thresholds ensures that the warn_threshold is lte error_threshold on custom thresholds', () => { + const config: Record = { + monitored_task_execution_thresholds: { + custom: { + 'alerting:always-fires': { + error_threshold: 80, + warn_threshold: 90, + }, + }, + }, + }; + expect(() => { + configSchema.validate(config); + }).toThrowErrorMatchingInlineSnapshot( + `"[monitored_task_execution_thresholds.custom.alerting:always-fires]: warn_threshold (90) must be less than, or equal to, error_threshold (80)"` + ); + }); + + test('the monitored_task_execution_thresholds allows a custom error_threshold which is lower than the default warn_threshold', () => { + const config: Record = { + monitored_task_execution_thresholds: { + default: { + warn_threshold: 80, + error_threshold: 90, + }, + custom: { + 'alerting:always-fires': { + error_threshold: 60, + warn_threshold: 50, + }, + }, + }, + }; + expect(() => { + configSchema.validate(config); + }).not.toThrowError(); + }); }); diff --git a/x-pack/plugins/task_manager/server/config.ts b/x-pack/plugins/task_manager/server/config.ts index aa78cf3baa96d4..157f01281836d6 100644 --- a/x-pack/plugins/task_manager/server/config.ts +++ b/x-pack/plugins/task_manager/server/config.ts @@ -10,44 +10,109 @@ export const DEFAULT_MAX_WORKERS = 10; export const DEFAULT_POLL_INTERVAL = 3000; export const DEFAULT_MAX_POLL_INACTIVITY_CYCLES = 10; -export const configSchema = schema.object({ - enabled: schema.boolean({ defaultValue: true }), - /* The maximum number of times a task will be attempted before being abandoned as failed */ - max_attempts: schema.number({ - defaultValue: 3, - min: 1, - }), - /* How often, in milliseconds, the task manager will look for more work. */ - poll_interval: schema.number({ - defaultValue: DEFAULT_POLL_INTERVAL, - min: 100, - }), - /* How many poll interval cycles can work take before it's timed out. */ - max_poll_inactivity_cycles: schema.number({ - defaultValue: DEFAULT_MAX_POLL_INACTIVITY_CYCLES, - min: 1, - }), - /* How many requests can Task Manager buffer before it rejects new requests. */ - request_capacity: schema.number({ - // a nice round contrived number, feel free to change as we learn how it behaves - defaultValue: 1000, - min: 1, - }), - /* The name of the index used to store task information. */ - index: schema.string({ - defaultValue: '.kibana_task_manager', - validate: (val) => { - if (val.toLowerCase() === '.tasks') { - return `"${val}" is an invalid Kibana Task Manager index, as it is already in use by the ElasticSearch Tasks Manager`; +// Monitoring Constants +// =================== +// Refresh aggregated monitored stats at a default rate of once a minute +export const DEFAULT_MONITORING_REFRESH_RATE = 60 * 1000; +export const DEFAULT_MONITORING_STATS_RUNNING_AVERGAE_WINDOW = 50; + +export const taskExecutionFailureThresholdSchema = schema.object( + { + error_threshold: schema.number({ + defaultValue: 90, + min: 0, + }), + warn_threshold: schema.number({ + defaultValue: 80, + min: 0, + }), + }, + { + validate(config) { + if (config.error_threshold < config.warn_threshold) { + return `warn_threshold (${config.warn_threshold}) must be less than, or equal to, error_threshold (${config.error_threshold})`; + } + }, + } +); + +export const configSchema = schema.object( + { + enabled: schema.boolean({ defaultValue: true }), + /* The maximum number of times a task will be attempted before being abandoned as failed */ + max_attempts: schema.number({ + defaultValue: 3, + min: 1, + }), + /* How often, in milliseconds, the task manager will look for more work. */ + poll_interval: schema.number({ + defaultValue: DEFAULT_POLL_INTERVAL, + min: 100, + }), + /* How many poll interval cycles can work take before it's timed out. */ + max_poll_inactivity_cycles: schema.number({ + defaultValue: DEFAULT_MAX_POLL_INACTIVITY_CYCLES, + min: 1, + }), + /* How many requests can Task Manager buffer before it rejects new requests. */ + request_capacity: schema.number({ + // a nice round contrived number, feel free to change as we learn how it behaves + defaultValue: 1000, + min: 1, + }), + /* The name of the index used to store task information. */ + index: schema.string({ + defaultValue: '.kibana_task_manager', + validate: (val) => { + if (val.toLowerCase() === '.tasks') { + return `"${val}" is an invalid Kibana Task Manager index, as it is already in use by the ElasticSearch Tasks Manager`; + } + }, + }), + /* The maximum number of tasks that this Kibana instance will run simultaneously. */ + max_workers: schema.number({ + defaultValue: DEFAULT_MAX_WORKERS, + // disable the task manager rather than trying to specify it with 0 workers + min: 1, + }), + /* The rate at which we emit fresh monitored stats. By default we'll use the poll_interval (+ a slight buffer) */ + monitored_stats_required_freshness: schema.number({ + defaultValue: (config?: unknown) => + ((config as { poll_interval: number })?.poll_interval ?? DEFAULT_POLL_INTERVAL) + 1000, + min: 100, + }), + /* The rate at which we refresh monitored stats that require aggregation queries against ES. */ + monitored_aggregated_stats_refresh_rate: schema.number({ + defaultValue: DEFAULT_MONITORING_REFRESH_RATE, + /* don't run monitored stat aggregations any faster than once every 5 seconds */ + min: 5000, + }), + /* The size of the running average window for monitored stats. */ + monitored_stats_running_average_window: schema.number({ + defaultValue: DEFAULT_MONITORING_STATS_RUNNING_AVERGAE_WINDOW, + max: 100, + min: 10, + }), + /* Task Execution result warn & error thresholds. */ + monitored_task_execution_thresholds: schema.object({ + default: taskExecutionFailureThresholdSchema, + custom: schema.recordOf(schema.string(), taskExecutionFailureThresholdSchema, { + defaultValue: {}, + }), + }), + }, + { + validate: (config) => { + if ( + config.monitored_stats_required_freshness && + config.poll_interval && + config.monitored_stats_required_freshness < config.poll_interval + ) { + return `The specified monitored_stats_required_freshness (${config.monitored_stats_required_freshness}) is invalid, as it is below the poll_interval (${config.poll_interval})`; } }, - }), - /* The maximum number of tasks that this Kibana instance will run simultaneously. */ - max_workers: schema.number({ - defaultValue: DEFAULT_MAX_WORKERS, - // disable the task manager rather than trying to specify it with 0 workers - min: 1, - }), -}); + } +); export type TaskManagerConfig = TypeOf; +export type TaskExecutionFailureThreshold = TypeOf; diff --git a/x-pack/plugins/task_manager/server/integration_tests/managed_configuration.test.ts b/x-pack/plugins/task_manager/server/integration_tests/managed_configuration.test.ts index 11f6ccc8818500..01326c73bd6804 100644 --- a/x-pack/plugins/task_manager/server/integration_tests/managed_configuration.test.ts +++ b/x-pack/plugins/task_manager/server/integration_tests/managed_configuration.test.ts @@ -30,7 +30,17 @@ describe('managed configuration', () => { max_attempts: 9, poll_interval: 3000, max_poll_inactivity_cycles: 10, + monitored_aggregated_stats_refresh_rate: 60000, + monitored_stats_required_freshness: 4000, + monitored_stats_running_average_window: 50, request_capacity: 1000, + monitored_task_execution_thresholds: { + default: { + error_threshold: 90, + warn_threshold: 80, + }, + custom: {}, + }, }); logger = context.logger.get('taskManager'); diff --git a/x-pack/plugins/task_manager/server/lib/bulk_operation_buffer.ts b/x-pack/plugins/task_manager/server/lib/bulk_operation_buffer.ts index 6df5b064f2792e..4de92ffc770301 100644 --- a/x-pack/plugins/task_manager/server/lib/bulk_operation_buffer.ts +++ b/x-pack/plugins/task_manager/server/lib/bulk_operation_buffer.ts @@ -4,11 +4,11 @@ * you may not use this file except in compliance with the Elastic License. */ +import { Logger } from 'src/core/server'; import { map } from 'lodash'; import { Subject, race, from } from 'rxjs'; import { bufferWhen, filter, bufferCount, flatMap, mapTo, first } from 'rxjs/operators'; import { either, Result, asOk, asErr, Ok, Err } from './result_type'; -import { Logger } from '../../../../../src/core/server'; export interface BufferOptions { bufferMaxDuration?: number; diff --git a/x-pack/plugins/task_manager/server/lib/correct_deprecated_fields.ts b/x-pack/plugins/task_manager/server/lib/correct_deprecated_fields.ts index 9e5f4b7c143a2c..a15682a9d3f382 100644 --- a/x-pack/plugins/task_manager/server/lib/correct_deprecated_fields.ts +++ b/x-pack/plugins/task_manager/server/lib/correct_deprecated_fields.ts @@ -4,8 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ +import { Logger } from 'src/core/server'; import { TaskInstance, TaskInstanceWithDeprecatedFields } from '../task'; -import { Logger } from '../../../../../src/core/server'; export function ensureDeprecatedFieldsAreCorrected( { id, taskType, interval, schedule, ...taskInstance }: TaskInstanceWithDeprecatedFields, diff --git a/x-pack/plugins/task_manager/server/lib/intervals.test.ts b/x-pack/plugins/task_manager/server/lib/intervals.test.ts index ac28b81eaf490f..e79694915f926d 100644 --- a/x-pack/plugins/task_manager/server/lib/intervals.test.ts +++ b/x-pack/plugins/task_manager/server/lib/intervals.test.ts @@ -7,13 +7,13 @@ import _ from 'lodash'; import sinon from 'sinon'; import { - assertValidInterval, + parseIntervalAsSecond, + parseIntervalAsMillisecond, intervalFromNow, intervalFromDate, - minutesFromNow, - minutesFromDate, secondsFromNow, secondsFromDate, + asInterval, } from './intervals'; let fakeTimer: sinon.SinonFakeTimers; @@ -25,32 +25,100 @@ beforeAll(() => { afterAll(() => fakeTimer.restore()); describe('taskIntervals', () => { - describe('assertValidInterval', () => { + describe('parseIntervalAsSecond', () => { test('it accepts intervals in the form `Nm`', () => { - expect(() => assertValidInterval(`${_.random(1, 1000)}m`)).not.toThrow(); + expect(() => parseIntervalAsSecond(`${_.random(1, 1000)}m`)).not.toThrow(); }); test('it accepts intervals in the form `Ns`', () => { - expect(() => assertValidInterval(`${_.random(1, 1000)}s`)).not.toThrow(); + expect(() => parseIntervalAsSecond(`${_.random(1, 1000)}s`)).not.toThrow(); }); test('it rejects 0 based intervals', () => { - expect(() => assertValidInterval('0m')).toThrow( + expect(() => parseIntervalAsSecond('0m')).toThrow( /Invalid interval "0m"\. Intervals must be of the form {number}m. Example: 5m/ ); - expect(() => assertValidInterval('0s')).toThrow( + expect(() => parseIntervalAsSecond('0s')).toThrow( /Invalid interval "0s"\. Intervals must be of the form {number}m. Example: 5m/ ); }); test('it rejects intervals are not of the form `Nm` or `Ns`', () => { - expect(() => assertValidInterval(`5m 2s`)).toThrow( + expect(() => parseIntervalAsSecond(`5m 2s`)).toThrow( /Invalid interval "5m 2s"\. Intervals must be of the form {number}m. Example: 5m/ ); - expect(() => assertValidInterval(`hello`)).toThrow( + expect(() => parseIntervalAsSecond(`hello`)).toThrow( /Invalid interval "hello"\. Intervals must be of the form {number}m. Example: 5m/ ); }); + + test('returns an interval as s', () => { + expect(parseIntervalAsSecond('5s')).toEqual(5); + expect(parseIntervalAsSecond('15s')).toEqual(15); + expect(parseIntervalAsSecond('20m')).toEqual(20 * 60); + expect(parseIntervalAsSecond('61m')).toEqual(61 * 60); + expect(parseIntervalAsSecond('90m')).toEqual(90 * 60); + }); + }); + + describe('parseIntervalAsMillisecond', () => { + test('it accepts intervals in the form `Nm`', () => { + expect(() => parseIntervalAsMillisecond(`${_.random(1, 1000)}m`)).not.toThrow(); + }); + + test('it accepts intervals in the form `Ns`', () => { + expect(() => parseIntervalAsMillisecond(`${_.random(1, 1000)}s`)).not.toThrow(); + }); + + test('it rejects 0 based intervals', () => { + expect(() => parseIntervalAsMillisecond('0m')).toThrow( + /Invalid interval "0m"\. Intervals must be of the form {number}m. Example: 5m/ + ); + expect(() => parseIntervalAsMillisecond('0s')).toThrow( + /Invalid interval "0s"\. Intervals must be of the form {number}m. Example: 5m/ + ); + }); + + test('it rejects intervals are not of the form `Nm` or `Ns`', () => { + expect(() => parseIntervalAsMillisecond(`5m 2s`)).toThrow( + /Invalid interval "5m 2s"\. Intervals must be of the form {number}m. Example: 5m/ + ); + expect(() => parseIntervalAsMillisecond(`hello`)).toThrow( + /Invalid interval "hello"\. Intervals must be of the form {number}m. Example: 5m/ + ); + }); + + test('returns an interval as ms', () => { + expect(parseIntervalAsMillisecond('5s')).toEqual(5 * 1000); + expect(parseIntervalAsMillisecond('15s')).toEqual(15 * 1000); + expect(parseIntervalAsMillisecond('20m')).toEqual(20 * 60 * 1000); + expect(parseIntervalAsMillisecond('61m')).toEqual(61 * 60 * 1000); + expect(parseIntervalAsMillisecond('90m')).toEqual(90 * 60 * 1000); + }); + }); + + describe('asInterval', () => { + test('returns a ms interval when ms duration can only divide by ms', () => { + expect(asInterval(500)).toEqual('500ms'); + expect(asInterval(1500)).toEqual('1500ms'); + expect(asInterval(1001)).toEqual('1001ms'); + expect(asInterval(2001)).toEqual('2001ms'); + expect(asInterval(61001)).toEqual('61001ms'); + expect(asInterval(90001)).toEqual('90001ms'); + }); + + test('returns a seconds interval when ms duration divides by seconds', () => { + expect(asInterval(1000)).toEqual('1s'); + expect(asInterval(2000)).toEqual('2s'); + expect(asInterval(61000)).toEqual('61s'); + expect(asInterval(99000)).toEqual('99s'); + expect(asInterval(90000)).toEqual('90s'); + }); + + test('returns a minutes interval when ms duration divides by minutes', () => { + expect(asInterval(60000)).toEqual('1m'); + expect(asInterval(120000)).toEqual('2m'); + }); }); describe('intervalFromNow', () => { @@ -125,25 +193,6 @@ describe('taskIntervals', () => { }); }); - describe('minutesFromNow', () => { - test('it returns the current date plus a number of minutes', () => { - const mins = _.random(1, 100); - const expected = Date.now() + mins * 60 * 1000; - const nextRun = minutesFromNow(mins).getTime(); - expect(nextRun).toEqual(expected); - }); - }); - - describe('minutesFromDate', () => { - test('it returns the given date plus a number of minutes', () => { - const originalDate = new Date(2019, 1, 1); - const mins = _.random(1, 100); - const expected = originalDate.valueOf() + mins * 60 * 1000; - const nextRun = minutesFromDate(originalDate, mins).getTime(); - expect(expected).toEqual(nextRun); - }); - }); - describe('secondsFromNow', () => { test('it returns the current date plus a number of seconds', () => { const secs = _.random(1, 100); diff --git a/x-pack/plugins/task_manager/server/lib/intervals.ts b/x-pack/plugins/task_manager/server/lib/intervals.ts index 9009be5f78220b..a28dfa62a501f5 100644 --- a/x-pack/plugins/task_manager/server/lib/intervals.ts +++ b/x-pack/plugins/task_manager/server/lib/intervals.ts @@ -4,6 +4,28 @@ * you may not use this file except in compliance with the Elastic License. */ +import { memoize } from 'lodash'; + +export enum IntervalCadence { + Minute = 'm', + Second = 's', +} +const VALID_CADENCE = new Set(Object.values(IntervalCadence)); +const CADENCE_IN_MS: Record = { + [IntervalCadence.Second]: 1000, + [IntervalCadence.Minute]: 60 * 1000, +}; + +function isCadence(cadence: IntervalCadence | string): cadence is IntervalCadence { + return VALID_CADENCE.has(cadence as IntervalCadence); +} + +export function asInterval(ms: number): string { + const secondsRemainder = ms % 1000; + const minutesRemainder = ms % 60000; + return secondsRemainder ? `${ms}ms` : minutesRemainder ? `${ms / 1000}s` : `${ms / 60000}m`; +} + /** * Returns a date that is the specified interval from now. Currently, * only minute-intervals and second-intervals are supported. @@ -14,14 +36,7 @@ export function intervalFromNow(interval?: string): Date | undefined { if (interval === undefined) { return; } - - assertValidInterval(interval); - - if (isSeconds(interval)) { - return secondsFromNow(parseInterval(interval)); - } - - return minutesFromNow(parseInterval(interval)); + return secondsFromNow(parseIntervalAsSecond(interval)); } /** @@ -35,37 +50,7 @@ export function intervalFromDate(date: Date, interval?: string): Date | undefine if (interval === undefined) { return; } - - assertValidInterval(interval); - - if (isSeconds(interval)) { - return secondsFromDate(date, parseInterval(interval)); - } - - return minutesFromDate(date, parseInterval(interval)); -} - -/** - * Returns a date that is mins minutes from now. - * - * @param mins The number of mintues from now - */ -export function minutesFromNow(mins: number): Date { - return minutesFromDate(new Date(), mins); -} - -/** - * Returns a date that is mins minutes from given date. - * - * @param date The date to add minutes to - * @param mins The number of mintues from given date - */ -export function minutesFromDate(date: Date, mins: number): Date { - const result = new Date(date.valueOf()); - - result.setMinutes(result.getMinutes() + mins); - - return result; + return secondsFromDate(date, parseIntervalAsSecond(interval)); } /** @@ -85,9 +70,7 @@ export function secondsFromNow(secs: number): Date { */ export function secondsFromDate(date: Date, secs: number): Date { const result = new Date(date.valueOf()); - result.setSeconds(result.getSeconds() + secs); - return result; } @@ -95,29 +78,22 @@ export function secondsFromDate(date: Date, secs: number): Date { * Verifies that the specified interval matches our expected format. * * @param {string} interval - An interval such as `5m` or `10s` + * @returns {number} The interval as seconds */ -export function assertValidInterval(interval: string) { - if (isMinutes(interval)) { - return interval; +export const parseIntervalAsSecond = memoize((interval: string): number => { + return Math.round(parseIntervalAsMillisecond(interval) / 1000); +}); + +export const parseIntervalAsMillisecond = memoize((interval: string): number => { + const numericAsStr: string = interval.slice(0, -1); + const numeric: number = parseInt(numericAsStr, 10); + const cadence: IntervalCadence | string = interval.slice(-1); + if (!isCadence(cadence) || isNaN(numeric) || numeric <= 0 || !isNumeric(numericAsStr)) { + throw new Error( + `Invalid interval "${interval}". Intervals must be of the form {number}m. Example: 5m.` + ); } + return numeric * CADENCE_IN_MS[cadence]; +}); - if (isSeconds(interval)) { - return interval; - } - - throw new Error( - `Invalid interval "${interval}". Intervals must be of the form {number}m. Example: 5m.` - ); -} - -function parseInterval(interval: string) { - return parseInt(interval, 10); -} - -function isMinutes(interval: string) { - return /^[1-9][0-9]*m$/.test(interval); -} - -function isSeconds(interval: string) { - return /^[1-9][0-9]*s$/.test(interval); -} +const isNumeric = (numAsStr: string) => /^\d+$/.test(numAsStr); diff --git a/x-pack/plugins/task_manager/server/monitoring/configuration_statistics.test.ts b/x-pack/plugins/task_manager/server/monitoring/configuration_statistics.test.ts new file mode 100644 index 00000000000000..f97861901b5b5b --- /dev/null +++ b/x-pack/plugins/task_manager/server/monitoring/configuration_statistics.test.ts @@ -0,0 +1,98 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Subject } from 'rxjs'; +import { take, bufferCount } from 'rxjs/operators'; +import { createConfigurationAggregator } from './configuration_statistics'; +import { TaskManagerConfig } from '../config'; + +describe('Configuration Statistics Aggregator', () => { + test('merges the static config with the merged configs', async () => { + const configuration: TaskManagerConfig = { + enabled: true, + max_workers: 10, + index: 'foo', + max_attempts: 9, + poll_interval: 6000000, + monitored_stats_required_freshness: 6000000, + max_poll_inactivity_cycles: 10, + request_capacity: 1000, + monitored_aggregated_stats_refresh_rate: 5000, + monitored_stats_running_average_window: 50, + monitored_task_execution_thresholds: { + default: { + error_threshold: 90, + warn_threshold: 80, + }, + custom: {}, + }, + }; + + const managedConfig = { + maxWorkersConfiguration$: new Subject(), + pollIntervalConfiguration$: new Subject(), + }; + + return new Promise(async (resolve, reject) => { + createConfigurationAggregator(configuration, managedConfig) + .pipe(take(3), bufferCount(3)) + .subscribe(([initial, updatedWorkers, updatedInterval]) => { + expect(initial.value).toEqual({ + max_workers: 10, + poll_interval: 6000000, + max_poll_inactivity_cycles: 10, + request_capacity: 1000, + monitored_aggregated_stats_refresh_rate: 5000, + monitored_stats_running_average_window: 50, + monitored_task_execution_thresholds: { + default: { + error_threshold: 90, + warn_threshold: 80, + }, + custom: {}, + }, + }); + + expect(updatedWorkers.value).toEqual({ + max_workers: 8, + poll_interval: 6000000, + max_poll_inactivity_cycles: 10, + request_capacity: 1000, + monitored_aggregated_stats_refresh_rate: 5000, + monitored_stats_running_average_window: 50, + monitored_task_execution_thresholds: { + default: { + error_threshold: 90, + warn_threshold: 80, + }, + custom: {}, + }, + }); + + expect(updatedInterval.value).toEqual({ + max_workers: 8, + poll_interval: 3000, + max_poll_inactivity_cycles: 10, + request_capacity: 1000, + monitored_aggregated_stats_refresh_rate: 5000, + monitored_stats_running_average_window: 50, + monitored_task_execution_thresholds: { + default: { + error_threshold: 90, + warn_threshold: 80, + }, + custom: {}, + }, + }); + resolve(); + }, reject); + + managedConfig.maxWorkersConfiguration$.next(8); + + managedConfig.pollIntervalConfiguration$.next(3000); + }); + }); +}); diff --git a/x-pack/plugins/task_manager/server/monitoring/configuration_statistics.ts b/x-pack/plugins/task_manager/server/monitoring/configuration_statistics.ts new file mode 100644 index 00000000000000..22b08bc5c88dbe --- /dev/null +++ b/x-pack/plugins/task_manager/server/monitoring/configuration_statistics.ts @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { combineLatest, of } from 'rxjs'; +import { pick, merge } from 'lodash'; +import { map, startWith } from 'rxjs/operators'; +import { AggregatedStatProvider } from './runtime_statistics_aggregator'; +import { TaskManagerConfig } from '../config'; +import { ManagedConfiguration } from '../lib/create_managed_configuration'; + +const CONFIG_FIELDS_TO_EXPOSE = [ + 'request_capacity', + 'max_poll_inactivity_cycles', + 'monitored_aggregated_stats_refresh_rate', + 'monitored_stats_running_average_window', + 'monitored_task_execution_thresholds', +] as const; + +export type ConfigStat = Pick< + TaskManagerConfig, + 'max_workers' | 'poll_interval' | typeof CONFIG_FIELDS_TO_EXPOSE[number] +>; + +export function createConfigurationAggregator( + config: TaskManagerConfig, + managedConfig: ManagedConfiguration +): AggregatedStatProvider { + return combineLatest([ + of(pick(config, ...CONFIG_FIELDS_TO_EXPOSE)), + managedConfig.pollIntervalConfiguration$.pipe( + startWith(config.poll_interval), + map>((pollInterval) => ({ + poll_interval: pollInterval, + })) + ), + managedConfig.maxWorkersConfiguration$.pipe( + startWith(config.max_workers), + map>((maxWorkers) => ({ + max_workers: maxWorkers, + })) + ), + ]).pipe( + map((configurations) => ({ + key: 'configuration', + value: merge({}, ...configurations), + })) + ); +} diff --git a/x-pack/plugins/task_manager/server/monitoring/index.ts b/x-pack/plugins/task_manager/server/monitoring/index.ts new file mode 100644 index 00000000000000..8e71ce2519a7c7 --- /dev/null +++ b/x-pack/plugins/task_manager/server/monitoring/index.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Logger } from 'src/core/server'; +import { Observable } from 'rxjs'; +import { TaskManagerConfig } from '../config'; +import { + MonitoringStats, + createAggregators, + createMonitoringStatsStream, +} from './monitoring_stats_stream'; +import { TaskStore } from '../task_store'; +import { TaskPollingLifecycle } from '../polling_lifecycle'; +import { ManagedConfiguration } from '../lib/create_managed_configuration'; + +export { + MonitoringStats, + HealthStatus, + RawMonitoringStats, + summarizeMonitoringStats, + createAggregators, + createMonitoringStatsStream, +} from './monitoring_stats_stream'; + +export function createMonitoringStats( + taskPollingLifecycle: TaskPollingLifecycle, + taskStore: TaskStore, + config: TaskManagerConfig, + managedConfig: ManagedConfiguration, + logger: Logger +): Observable { + return createMonitoringStatsStream( + createAggregators(taskPollingLifecycle, taskStore, config, managedConfig, logger), + config + ); +} diff --git a/x-pack/plugins/task_manager/server/monitoring/monitoring_stats_stream.test.ts b/x-pack/plugins/task_manager/server/monitoring/monitoring_stats_stream.test.ts new file mode 100644 index 00000000000000..8479def5deeebb --- /dev/null +++ b/x-pack/plugins/task_manager/server/monitoring/monitoring_stats_stream.test.ts @@ -0,0 +1,173 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { TaskManagerConfig } from '../config'; +import { of, Subject } from 'rxjs'; +import { take, bufferCount } from 'rxjs/operators'; +import { createMonitoringStatsStream, AggregatedStat } from './monitoring_stats_stream'; +import { JsonValue } from 'src/plugins/kibana_utils/common'; + +beforeEach(() => { + jest.resetAllMocks(); +}); + +describe('createMonitoringStatsStream', () => { + const configuration: TaskManagerConfig = { + enabled: true, + max_workers: 10, + index: 'foo', + max_attempts: 9, + poll_interval: 6000000, + monitored_stats_required_freshness: 6000000, + max_poll_inactivity_cycles: 10, + request_capacity: 1000, + monitored_aggregated_stats_refresh_rate: 5000, + monitored_stats_running_average_window: 50, + monitored_task_execution_thresholds: { + default: { + error_threshold: 90, + warn_threshold: 80, + }, + custom: {}, + }, + }; + + it('returns the initial config used to configure Task Manager', async () => { + return new Promise((resolve) => { + createMonitoringStatsStream(of(), configuration) + .pipe(take(1)) + .subscribe((firstValue) => { + expect(firstValue.stats).toEqual({}); + resolve(); + }); + }); + }); + + it('incrementally updates the stats returned by the endpoint', async () => { + const aggregatedStats$ = new Subject(); + + return new Promise((resolve) => { + createMonitoringStatsStream(aggregatedStats$, configuration) + .pipe(take(3), bufferCount(3)) + .subscribe(([initialValue, secondValue, thirdValue]) => { + expect(initialValue.stats).toMatchObject({ + lastUpdate: expect.any(String), + stats: { + configuration: { + value: { + max_workers: 10, + poll_interval: 6000000, + max_poll_inactivity_cycles: 10, + request_capacity: 1000, + monitored_aggregated_stats_refresh_rate: 5000, + monitored_stats_running_average_window: 50, + monitored_task_execution_thresholds: { + default: { + error_threshold: 90, + warn_threshold: 80, + }, + custom: {}, + }, + }, + }, + }, + }); + + expect(secondValue).toMatchObject({ + lastUpdate: expect.any(String), + stats: { + newAggregatedStat: { + timestamp: expect.any(String), + value: { + some: { + complex: { + value: 123, + }, + }, + }, + }, + configuration: { + timestamp: expect.any(String), + value: { + max_workers: 10, + poll_interval: 6000000, + max_poll_inactivity_cycles: 10, + request_capacity: 1000, + monitored_aggregated_stats_refresh_rate: 5000, + monitored_stats_running_average_window: 50, + monitored_task_execution_thresholds: { + default: { + error_threshold: 90, + warn_threshold: 80, + }, + custom: {}, + }, + }, + }, + }, + }); + + expect(thirdValue).toMatchObject({ + lastUpdate: expect.any(String), + stats: { + newAggregatedStat: { + timestamp: expect.any(String), + value: { + some: { + updated: { + value: 456, + }, + }, + }, + }, + configuration: { + timestamp: expect.any(String), + value: { + max_workers: 10, + poll_interval: 6000000, + max_poll_inactivity_cycles: 10, + request_capacity: 1000, + monitored_aggregated_stats_refresh_rate: 5000, + monitored_stats_running_average_window: 50, + monitored_task_execution_thresholds: { + default: { + error_threshold: 90, + warn_threshold: 80, + }, + custom: {}, + }, + }, + }, + }, + }); + }); + + aggregatedStats$.next({ + key: 'newAggregatedStat', + value: { + some: { + complex: { + value: 123, + }, + }, + } as JsonValue, + }); + + aggregatedStats$.next({ + key: 'newAggregatedStat', + value: { + some: { + updated: { + value: 456, + }, + }, + } as JsonValue, + }); + + resolve(); + }); + }); +}); diff --git a/x-pack/plugins/task_manager/server/monitoring/monitoring_stats_stream.ts b/x-pack/plugins/task_manager/server/monitoring/monitoring_stats_stream.ts new file mode 100644 index 00000000000000..374660a257c593 --- /dev/null +++ b/x-pack/plugins/task_manager/server/monitoring/monitoring_stats_stream.ts @@ -0,0 +1,148 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { merge, of, Observable } from 'rxjs'; +import { map, scan } from 'rxjs/operators'; +import { set } from '@elastic/safer-lodash-set'; +import { Logger } from 'src/core/server'; +import { JsonObject } from 'src/plugins/kibana_utils/common'; +import { TaskStore } from '../task_store'; +import { TaskPollingLifecycle } from '../polling_lifecycle'; +import { + createWorkloadAggregator, + summarizeWorkloadStat, + WorkloadStat, +} from './workload_statistics'; +import { + createTaskRunAggregator, + summarizeTaskRunStat, + TaskRunStat, + SummarizedTaskRunStat, +} from './task_run_statistics'; +import { ConfigStat, createConfigurationAggregator } from './configuration_statistics'; +import { TaskManagerConfig } from '../config'; +import { AggregatedStatProvider } from './runtime_statistics_aggregator'; +import { ManagedConfiguration } from '../lib/create_managed_configuration'; + +export { AggregatedStatProvider, AggregatedStat } from './runtime_statistics_aggregator'; + +export interface MonitoringStats { + last_update: string; + stats: { + configuration?: MonitoredStat; + workload?: MonitoredStat; + runtime?: MonitoredStat; + }; +} + +export enum HealthStatus { + OK = 'OK', + Warning = 'warn', + Error = 'error', +} + +interface MonitoredStat { + timestamp: string; + value: T; +} +type RawMonitoredStat = MonitoredStat & { + status: HealthStatus; +}; + +export interface RawMonitoringStats { + last_update: string; + stats: { + configuration?: RawMonitoredStat; + workload?: RawMonitoredStat; + runtime?: RawMonitoredStat; + }; +} + +export function createAggregators( + taskPollingLifecycle: TaskPollingLifecycle, + taskStore: TaskStore, + config: TaskManagerConfig, + managedConfig: ManagedConfiguration, + logger: Logger +): AggregatedStatProvider { + return merge( + createConfigurationAggregator(config, managedConfig), + createTaskRunAggregator(taskPollingLifecycle, config.monitored_stats_running_average_window), + createWorkloadAggregator( + taskStore, + config.monitored_aggregated_stats_refresh_rate, + config.poll_interval, + logger + ) + ); +} + +export function createMonitoringStatsStream( + provider$: AggregatedStatProvider, + config: TaskManagerConfig +): Observable { + const initialStats = { + last_update: new Date().toISOString(), + stats: {}, + }; + return merge( + // emit the initial stats + of(initialStats), + // emit updated stats whenever a provider updates a specific key on the stats + provider$.pipe( + map(({ key, value }) => { + return { + value: { timestamp: new Date().toISOString(), value }, + key, + }; + }), + scan((monitoringStats: MonitoringStats, { key, value }) => { + // incrementally merge stats as they come in + set(monitoringStats.stats, key, value); + monitoringStats.last_update = new Date().toISOString(); + return monitoringStats; + }, initialStats) + ) + ); +} + +export function summarizeMonitoringStats( + { + // eslint-disable-next-line @typescript-eslint/naming-convention + last_update, + stats: { runtime, workload, configuration }, + }: MonitoringStats, + config: TaskManagerConfig +): RawMonitoringStats { + return { + last_update, + stats: { + ...(configuration + ? { + configuration: { + ...configuration, + status: HealthStatus.OK, + }, + } + : {}), + ...(runtime + ? { + runtime: { + timestamp: runtime.timestamp, + ...summarizeTaskRunStat(runtime.value, config), + }, + } + : {}), + ...(workload + ? { + workload: { + timestamp: workload.timestamp, + ...summarizeWorkloadStat(workload.value), + }, + } + : {}), + }, + }; +} diff --git a/x-pack/plugins/task_manager/server/monitoring/runtime_statistics_aggregator.ts b/x-pack/plugins/task_manager/server/monitoring/runtime_statistics_aggregator.ts new file mode 100644 index 00000000000000..bd2b3845f2526d --- /dev/null +++ b/x-pack/plugins/task_manager/server/monitoring/runtime_statistics_aggregator.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { Observable } from 'rxjs'; +import { JsonValue } from 'src/plugins/kibana_utils/common'; + +export interface AggregatedStat { + key: string; + value: Stat; +} + +export type AggregatedStatProvider = Observable< + AggregatedStat +>; diff --git a/x-pack/plugins/task_manager/server/monitoring/task_run_calcultors.test.ts b/x-pack/plugins/task_manager/server/monitoring/task_run_calcultors.test.ts new file mode 100644 index 00000000000000..eb8cabd9f3a8f6 --- /dev/null +++ b/x-pack/plugins/task_manager/server/monitoring/task_run_calcultors.test.ts @@ -0,0 +1,79 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import uuid from 'uuid'; + +import { + calculateRunningAverage, + calculateFrequency, + createRunningAveragedStat, + createMapOfRunningAveragedStats, +} from './task_run_calcultors'; + +describe('calculateRunningAverage', () => { + test('calculates the running average and median of a window of values', async () => { + expect(calculateRunningAverage([2, 2, 4, 6, 6])).toMatchInlineSnapshot(` + Object { + "p50": 4, + "p90": 6, + "p95": 6, + "p99": 6, + } + `); + }); +}); + +describe('calculateFrequency', () => { + test('calculates the frequency of each terms in the list as a percentage', async () => { + const [term1, term2, term3] = [uuid.v4(), uuid.v4(), uuid.v4()]; + expect( + calculateFrequency([term1, term2, term2, term3, term1, term1, term2, term1, term3]) + ).toEqual({ + [term3]: 22, + [term1]: 44, + [term2]: 33, + }); + }); +}); + +describe('createRunningAveragedStat', () => { + test('create a function which tracks a window of values', async () => { + const queue = createRunningAveragedStat(3); + expect(queue(1)).toEqual([1]); + expect(queue(2)).toEqual([1, 2]); + expect(queue(3)).toEqual([1, 2, 3]); + expect(queue(4)).toEqual([2, 3, 4]); + expect(queue(5)).toEqual([3, 4, 5]); + }); +}); + +describe('createMapOfRunningAveragedStats', () => { + test('create a function which tracks multiple window of values by key', async () => { + const [term1, term2, term3] = [uuid.v4(), uuid.v4(), uuid.v4()]; + const mappedQueues = createMapOfRunningAveragedStats(3); + expect(mappedQueues(term1, 1)).toEqual({ [term1]: [1] }); + expect(mappedQueues(term1, 2)).toEqual({ [term1]: [1, 2] }); + expect(mappedQueues(term2, 3)).toEqual({ [term1]: [1, 2], [term2]: [3] }); + expect(mappedQueues(term3, 4)).toEqual({ [term1]: [1, 2], [term2]: [3], [term3]: [4] }); + expect(mappedQueues(term2, 5)).toEqual({ [term1]: [1, 2], [term2]: [3, 5], [term3]: [4] }); + expect(mappedQueues(term2, 6)).toEqual({ [term1]: [1, 2], [term2]: [3, 5, 6], [term3]: [4] }); + expect(mappedQueues(term1, 7)).toEqual({ + [term1]: [1, 2, 7], + [term2]: [3, 5, 6], + [term3]: [4], + }); + expect(mappedQueues(term1, 8)).toEqual({ + [term1]: [2, 7, 8], + [term2]: [3, 5, 6], + [term3]: [4], + }); + expect(mappedQueues(term1, 9)).toEqual({ + [term1]: [7, 8, 9], + [term2]: [3, 5, 6], + [term3]: [4], + }); + }); +}); diff --git a/x-pack/plugins/task_manager/server/monitoring/task_run_calcultors.ts b/x-pack/plugins/task_manager/server/monitoring/task_run_calcultors.ts new file mode 100644 index 00000000000000..67b77a29b1c76a --- /dev/null +++ b/x-pack/plugins/task_manager/server/monitoring/task_run_calcultors.ts @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import stats from 'stats-lite'; +import { JsonObject } from 'src/plugins/kibana_utils/common'; +import { isUndefined, countBy, mapValues } from 'lodash'; + +export interface AveragedStat extends JsonObject { + p50: number; + p90: number; + p95: number; + p99: number; +} + +export function calculateRunningAverage(values: number[]): AveragedStat { + return { + p50: stats.percentile(values, 0.5), + p90: stats.percentile(values, 0.9), + p95: stats.percentile(values, 0.95), + p99: stats.percentile(values, 0.99), + }; +} + +/** + * Calculate the frequency of each term in a list of terms. + * @param values + */ +export function calculateFrequency(values: T[]): JsonObject { + return values.length + ? mapValues(countBy(values), (count) => Math.round((count * 100) / values.length)) + : {}; +} + +/** + * Utility to keep track of a bounded array of values which changes over time + * dropping older values as they slide out of the window we wish to track + */ +export function createRunningAveragedStat(runningAverageWindowSize: number) { + const list = new Array(); + return (value?: T) => { + if (!isUndefined(value)) { + if (list.length === runningAverageWindowSize) { + list.shift(); + } + list.push(value); + } + // clone list to ensure it isn't mutated externally + return [...list]; + }; +} + +export function createMapOfRunningAveragedStats(runningAverageWindowSize: number) { + const mappedQueue: Record T[]> = {}; + const asRecordOfValues = () => mapValues(mappedQueue, (queue) => queue()); + return (key?: string, value?: T) => { + if (!isUndefined(key)) { + mappedQueue[key] = mappedQueue[key] ?? createRunningAveragedStat(runningAverageWindowSize); + mappedQueue[key](value); + } + return asRecordOfValues(); + }; +} diff --git a/x-pack/plugins/task_manager/server/monitoring/task_run_statistics.test.ts b/x-pack/plugins/task_manager/server/monitoring/task_run_statistics.test.ts new file mode 100644 index 00000000000000..a931f0ff7c3048 --- /dev/null +++ b/x-pack/plugins/task_manager/server/monitoring/task_run_statistics.test.ts @@ -0,0 +1,501 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import uuid from 'uuid'; +import { Subject, Observable } from 'rxjs'; +import stats from 'stats-lite'; +import sinon from 'sinon'; +import { take, tap, bufferCount, skip, map } from 'rxjs/operators'; + +import { ConcreteTaskInstance, TaskStatus } from '../task'; +import { asTaskRunEvent, asTaskPollingCycleEvent, TaskTiming } from '../task_events'; +import { asOk } from '../lib/result_type'; +import { TaskLifecycleEvent } from '../polling_lifecycle'; +import { TaskRunResult } from '../task_runner'; +import { + createTaskRunAggregator, + summarizeTaskRunStat, + TaskRunStat, + SummarizedTaskRunStat, +} from './task_run_statistics'; +import { AggregatedStat } from './runtime_statistics_aggregator'; +import { FillPoolResult } from '../lib/fill_pool'; +import { taskPollingLifecycleMock } from '../polling_lifecycle.mock'; +import { configSchema } from '../config'; + +describe('Task Run Statistics', () => { + let fakeTimer: sinon.SinonFakeTimers; + + beforeAll(() => { + fakeTimer = sinon.useFakeTimers(); + }); + + afterAll(() => fakeTimer.restore()); + + test('returns a running average of task drift', async () => { + const runAtDrift = [1000, 2000, 500, 300, 400, 15000, 20000, 200]; + const events$ = new Subject(); + const taskPollingLifecycle = taskPollingLifecycleMock.create({ + events$: events$ as Observable, + }); + + const runningAverageWindowSize = 5; + const taskRunAggregator = createTaskRunAggregator( + taskPollingLifecycle, + runningAverageWindowSize + ); + + function expectWindowEqualsUpdate( + taskStat: AggregatedStat, + window: number[] + ) { + expect(taskStat.value.drift).toMatchObject({ + p50: stats.percentile(window, 0.5), + p90: stats.percentile(window, 0.9), + p95: stats.percentile(window, 0.95), + p99: stats.percentile(window, 0.99), + }); + } + + return new Promise((resolve) => { + taskRunAggregator + .pipe( + // skip initial stat which is just initialized data which + // ensures we don't stall on combineLatest + skip(1), + // Use 'summarizeTaskRunStat' to receive summarize stats + map(({ key, value }: AggregatedStat) => ({ + key, + value: summarizeTaskRunStat(value, getTaskManagerConfig()).value, + })), + take(runAtDrift.length), + bufferCount(runAtDrift.length) + ) + .subscribe((taskStats: Array>) => { + expectWindowEqualsUpdate(taskStats[0], runAtDrift.slice(0, 1)); + expectWindowEqualsUpdate(taskStats[1], runAtDrift.slice(0, 2)); + expectWindowEqualsUpdate(taskStats[2], runAtDrift.slice(0, 3)); + expectWindowEqualsUpdate(taskStats[3], runAtDrift.slice(0, 4)); + expectWindowEqualsUpdate(taskStats[4], runAtDrift.slice(0, 5)); + // from the 6th value, begin to drop old values as out window is 5 + expectWindowEqualsUpdate(taskStats[5], runAtDrift.slice(1, 6)); + expectWindowEqualsUpdate(taskStats[6], runAtDrift.slice(2, 7)); + expectWindowEqualsUpdate(taskStats[7], runAtDrift.slice(3, 8)); + resolve(); + }); + + const now = Date.now(); + for (const drift of runAtDrift) { + const start = Math.floor(Math.random() * 1000); + events$.next( + mockTaskRunEvent( + { runAt: runAtMillisecondsAgo(drift + start) }, + { start: runAtMillisecondsAgo(start).getTime(), stop: now } + ) + ); + } + }); + }); + + test('returns a running average of task run duration', async () => { + const runDurations = [1000, 2000, 500, 300, 400, 15000, 20000, 200]; + const runDurationsInReverse = runDurations.reverse(); + const events$ = new Subject(); + const taskPollingLifecycle = taskPollingLifecycleMock.create({ + events$: events$ as Observable, + }); + + const runningAverageWindowSize = 5; + const taskRunAggregator = createTaskRunAggregator( + taskPollingLifecycle, + runningAverageWindowSize + ); + + function expectWindowEqualsUpdate( + taskStat: AggregatedStat, + windows: Record + ) { + for (const [type, window] of Object.entries(windows)) { + expect(taskStat.value.execution.duration[type]).toMatchObject({ + p50: stats.percentile(window, 0.5), + p90: stats.percentile(window, 0.9), + p95: stats.percentile(window, 0.95), + p99: stats.percentile(window, 0.99), + }); + } + } + + return new Promise((resolve, reject) => { + taskRunAggregator + .pipe( + // skip initial stat which is just initialized data which + // ensures we don't stall on combineLatest + skip(1), + // Use 'summarizeTaskRunStat' to receive summarize stats + map(({ key, value }: AggregatedStat) => ({ + key, + value: summarizeTaskRunStat(value, getTaskManagerConfig()).value, + })), + take(runDurations.length * 2), + bufferCount(runDurations.length * 2) + ) + .subscribe((taskStats: Array>) => { + try { + expectWindowEqualsUpdate(taskStats[0], { 'alerting:test': runDurations.slice(0, 1) }); + expectWindowEqualsUpdate(taskStats[1], { 'alerting:test': runDurations.slice(0, 2) }); + expectWindowEqualsUpdate(taskStats[2], { 'alerting:test': runDurations.slice(0, 3) }); + expectWindowEqualsUpdate(taskStats[3], { 'alerting:test': runDurations.slice(0, 4) }); + expectWindowEqualsUpdate(taskStats[4], { 'alerting:test': runDurations.slice(0, 5) }); + // from the 6th value, begin to drop old values as out window is 5 + expectWindowEqualsUpdate(taskStats[5], { 'alerting:test': runDurations.slice(1, 6) }); + expectWindowEqualsUpdate(taskStats[6], { 'alerting:test': runDurations.slice(2, 7) }); + expectWindowEqualsUpdate(taskStats[7], { 'alerting:test': runDurations.slice(3, 8) }); + expectWindowEqualsUpdate(taskStats[8], { + 'actions:test': runDurations.slice(0, 1), + 'alerting:test': runDurations.slice(3, 8), + }); + expectWindowEqualsUpdate(taskStats[9], { + 'actions:test': runDurations.slice(0, 2), + 'alerting:test': runDurations.slice(3, 8), + }); + expectWindowEqualsUpdate(taskStats[10], { + 'actions:test': runDurations.slice(0, 3), + 'alerting:test': runDurations.slice(3, 8), + }); + expectWindowEqualsUpdate(taskStats[11], { + 'actions:test': runDurations.slice(0, 4), + 'alerting:test': runDurations.slice(3, 8), + }); + expectWindowEqualsUpdate(taskStats[12], { + 'actions:test': runDurations.slice(0, 5), + 'alerting:test': runDurations.slice(3, 8), + }); + // from the 6th value, begin to drop old values as out window is 5 + expectWindowEqualsUpdate(taskStats[13], { + 'actions:test': runDurations.slice(1, 6), + 'alerting:test': runDurations.slice(3, 8), + }); + expectWindowEqualsUpdate(taskStats[14], { + 'actions:test': runDurations.slice(2, 7), + 'alerting:test': runDurations.slice(3, 8), + }); + expectWindowEqualsUpdate(taskStats[15], { + 'actions:test': runDurations.slice(3, 8), + 'alerting:test': runDurations.slice(3, 8), + }); + resolve(); + } catch (e) { + reject(e); + } + }); + + const now = Date.now(); + for (const runDuration of runDurations) { + events$.next( + mockTaskRunEvent( + { taskType: 'alerting:test' }, + { start: runAtMillisecondsAgo(runDuration).getTime(), stop: now } + ) + ); + } + for (const runDuration of runDurationsInReverse) { + events$.next( + mockTaskRunEvent( + { taskType: 'actions:test' }, + { start: runAtMillisecondsAgo(runDuration).getTime(), stop: now } + ) + ); + } + }); + }); + + test('returns the frequency of task run results', async () => { + const events$ = new Subject(); + const taskPollingLifecycle = taskPollingLifecycleMock.create({ + events$: events$ as Observable, + }); + + const runningAverageWindowSize = 5; + const taskRunAggregator = createTaskRunAggregator( + taskPollingLifecycle, + runningAverageWindowSize + ); + + return new Promise((resolve, reject) => { + taskRunAggregator + .pipe( + // skip initial stat which is just initialized data which + // ensures we don't stall on combineLatest + skip(1), + // Use 'summarizeTaskRunStat' to receive summarize stats + map(({ key, value }: AggregatedStat) => ({ + key, + value: summarizeTaskRunStat(value, getTaskManagerConfig()).value, + })), + take(10), + bufferCount(10) + ) + .subscribe((taskStats: Array>) => { + try { + /** + * At any given time we only keep track of the last X Polling Results + * In the tests this is ocnfiugured to a window size of 5 + */ + expect( + taskStats.map( + (taskStat) => + taskStat.value.execution.result_frequency_percent_as_number['alerting:test'] + ) + ).toEqual([ + // Success + { Success: 100, RetryScheduled: 0, Failed: 0, status: 'OK' }, + // Success, Success, + { Success: 100, RetryScheduled: 0, Failed: 0, status: 'OK' }, + // Success, Success, Success + { Success: 100, RetryScheduled: 0, Failed: 0, status: 'OK' }, + // Success, Success, Success, Failed + { Success: 75, RetryScheduled: 0, Failed: 25, status: 'OK' }, + // Success, Success, Success, Failed, Failed + { Success: 60, RetryScheduled: 0, Failed: 40, status: 'OK' }, + // Success, Success, Failed, Failed, Failed + { Success: 40, RetryScheduled: 0, Failed: 60, status: 'OK' }, + // Success, Failed, Failed, Failed, RetryScheduled + { Success: 20, RetryScheduled: 20, Failed: 60, status: 'OK' }, + // Failed, Failed, Failed, RetryScheduled, RetryScheduled + { Success: 0, RetryScheduled: 40, Failed: 60, status: 'OK' }, + // Failed, Failed, RetryScheduled, RetryScheduled, Success + { Success: 20, RetryScheduled: 40, Failed: 40, status: 'OK' }, + // Failed, RetryScheduled, RetryScheduled, Success, Success + { Success: 40, RetryScheduled: 40, Failed: 20, status: 'OK' }, + ]); + resolve(); + } catch (e) { + reject(e); + } + }); + + events$.next(mockTaskRunEvent({}, { start: 0, stop: 0 }, TaskRunResult.Success)); + events$.next(mockTaskRunEvent({}, { start: 0, stop: 0 }, TaskRunResult.Success)); + events$.next(mockTaskRunEvent({}, { start: 0, stop: 0 }, TaskRunResult.Success)); + events$.next(mockTaskRunEvent({}, { start: 0, stop: 0 }, TaskRunResult.Failed)); + events$.next(mockTaskRunEvent({}, { start: 0, stop: 0 }, TaskRunResult.Failed)); + events$.next(mockTaskRunEvent({}, { start: 0, stop: 0 }, TaskRunResult.Failed)); + events$.next(mockTaskRunEvent({}, { start: 0, stop: 0 }, TaskRunResult.RetryScheduled)); + events$.next(mockTaskRunEvent({}, { start: 0, stop: 0 }, TaskRunResult.RetryScheduled)); + events$.next(mockTaskRunEvent({}, { start: 0, stop: 0 }, TaskRunResult.Success)); + events$.next(mockTaskRunEvent({}, { start: 0, stop: 0 }, TaskRunResult.Success)); + }); + }); + + test('frequency of task run results return an error health status when failure is above a certain threshold', async () => { + const events$ = new Subject(); + + const taskPollingLifecycle = taskPollingLifecycleMock.create({ + events$: events$ as Observable, + }); + + const runningAverageWindowSize = 5; + const taskRunAggregator = createTaskRunAggregator( + taskPollingLifecycle, + runningAverageWindowSize + ); + + return new Promise((resolve, reject) => { + taskRunAggregator + .pipe( + // skip initial stat which is just initialized data which + // ensures we don't stall on combineLatest + skip(1), + // Use 'summarizeTaskRunStat' to receive summarize stats + map(({ key, value }: AggregatedStat) => ({ + key, + value: summarizeTaskRunStat( + value, + getTaskManagerConfig({ + monitored_task_execution_thresholds: { + custom: { + 'alerting:test': { + error_threshold: 59, + warn_threshold: 39, + }, + }, + }, + }) + ).value, + })), + take(10), + bufferCount(10) + ) + .subscribe((taskStats: Array>) => { + try { + /** + * At any given time we only keep track of the last X Polling Results + * In the tests this is ocnfiugured to a window size of 5 + */ + expect( + taskStats.map( + (taskStat) => + taskStat.value.execution.result_frequency_percent_as_number['alerting:test'] + ) + ).toEqual([ + // Success + { Success: 100, RetryScheduled: 0, Failed: 0, status: 'OK' }, + // Success, Success, + { Success: 100, RetryScheduled: 0, Failed: 0, status: 'OK' }, + // Success, Success, Success + { Success: 100, RetryScheduled: 0, Failed: 0, status: 'OK' }, + // Success, Success, Success, Failed + { Success: 75, RetryScheduled: 0, Failed: 25, status: 'OK' }, + // Success, Success, Success, Failed, Failed + { Success: 60, RetryScheduled: 0, Failed: 40, status: 'warn' }, + // Success, Success, Failed, Failed, Failed + { Success: 40, RetryScheduled: 0, Failed: 60, status: 'error' }, + // Success, Failed, Failed, Failed, RetryScheduled + { Success: 20, RetryScheduled: 20, Failed: 60, status: 'error' }, + // Failed, Failed, Failed, RetryScheduled, RetryScheduled + { Success: 0, RetryScheduled: 40, Failed: 60, status: 'error' }, + // Failed, Failed, RetryScheduled, RetryScheduled, Success + { Success: 20, RetryScheduled: 40, Failed: 40, status: 'warn' }, + // Failed, RetryScheduled, RetryScheduled, Success, Success + { Success: 40, RetryScheduled: 40, Failed: 20, status: 'OK' }, + ]); + resolve(); + } catch (e) { + reject(e); + } + }); + + events$.next(mockTaskRunEvent({}, { start: 0, stop: 0 }, TaskRunResult.Success)); + events$.next(mockTaskRunEvent({}, { start: 0, stop: 0 }, TaskRunResult.Success)); + events$.next(mockTaskRunEvent({}, { start: 0, stop: 0 }, TaskRunResult.Success)); + events$.next(mockTaskRunEvent({}, { start: 0, stop: 0 }, TaskRunResult.Failed)); + events$.next(mockTaskRunEvent({}, { start: 0, stop: 0 }, TaskRunResult.Failed)); + events$.next(mockTaskRunEvent({}, { start: 0, stop: 0 }, TaskRunResult.Failed)); + events$.next(mockTaskRunEvent({}, { start: 0, stop: 0 }, TaskRunResult.RetryScheduled)); + events$.next(mockTaskRunEvent({}, { start: 0, stop: 0 }, TaskRunResult.RetryScheduled)); + events$.next(mockTaskRunEvent({}, { start: 0, stop: 0 }, TaskRunResult.Success)); + events$.next(mockTaskRunEvent({}, { start: 0, stop: 0 }, TaskRunResult.Success)); + }); + }); + + test('returns polling stats', async () => { + const expectedTimestamp: string[] = []; + const events$ = new Subject(); + const taskPollingLifecycle = taskPollingLifecycleMock.create({ + events$: events$ as Observable, + }); + + const runningAverageWindowSize = 5; + const taskRunAggregator = createTaskRunAggregator( + taskPollingLifecycle, + runningAverageWindowSize + ); + + return new Promise((resolve, reject) => { + taskRunAggregator + .pipe( + // skip initial stat which is just initialized data which + // ensures we don't stall on combineLatest + skip(1), + // Use 'summarizeTaskRunStat' to receive summarize stats + map(({ key, value }: AggregatedStat) => ({ + key, + value: summarizeTaskRunStat(value, getTaskManagerConfig()).value, + })), + tap(() => { + expectedTimestamp.push(new Date().toISOString()); + // each event is a second after the previous one + fakeTimer.tick(1000); + }), + take(10), + bufferCount(10) + ) + .subscribe((taskStats: Array>) => { + try { + expect( + taskStats.map((taskStat) => taskStat.value.polling.last_successful_poll) + ).toEqual(expectedTimestamp); + + /** + * At any given time we only keep track of the last X Polling Results + * In the tests this is ocnfiugured to a window size of 5 + */ + expect( + taskStats.map((taskStat) => taskStat.value.polling.result_frequency_percent_as_number) + ).toEqual([ + // NoTasksClaimed + { NoTasksClaimed: 100, RanOutOfCapacity: 0, PoolFilled: 0 }, + // NoTasksClaimed, NoTasksClaimed, + { NoTasksClaimed: 100, RanOutOfCapacity: 0, PoolFilled: 0 }, + // NoTasksClaimed, NoTasksClaimed, NoTasksClaimed + { NoTasksClaimed: 100, RanOutOfCapacity: 0, PoolFilled: 0 }, + // NoTasksClaimed, NoTasksClaimed, NoTasksClaimed, PoolFilled + { NoTasksClaimed: 75, RanOutOfCapacity: 0, PoolFilled: 25 }, + // NoTasksClaimed, NoTasksClaimed, NoTasksClaimed, PoolFilled, PoolFilled + { NoTasksClaimed: 60, RanOutOfCapacity: 0, PoolFilled: 40 }, + // NoTasksClaimed, NoTasksClaimed, PoolFilled, PoolFilled, PoolFilled + { NoTasksClaimed: 40, RanOutOfCapacity: 0, PoolFilled: 60 }, + // NoTasksClaimed, PoolFilled, PoolFilled, PoolFilled, RanOutOfCapacity + { NoTasksClaimed: 20, RanOutOfCapacity: 20, PoolFilled: 60 }, + // PoolFilled, PoolFilled, PoolFilled, RanOutOfCapacity, RanOutOfCapacity + { NoTasksClaimed: 0, RanOutOfCapacity: 40, PoolFilled: 60 }, + // PoolFilled, PoolFilled, RanOutOfCapacity, RanOutOfCapacity, NoTasksClaimed + { NoTasksClaimed: 20, RanOutOfCapacity: 40, PoolFilled: 40 }, + // PoolFilled, RanOutOfCapacity, RanOutOfCapacity, NoTasksClaimed, NoTasksClaimed + { NoTasksClaimed: 40, RanOutOfCapacity: 40, PoolFilled: 20 }, + ]); + resolve(); + } catch (e) { + reject(e); + } + }); + + events$.next(asTaskPollingCycleEvent(asOk(FillPoolResult.NoTasksClaimed))); + events$.next(asTaskPollingCycleEvent(asOk(FillPoolResult.NoTasksClaimed))); + events$.next(asTaskPollingCycleEvent(asOk(FillPoolResult.NoTasksClaimed))); + events$.next(asTaskPollingCycleEvent(asOk(FillPoolResult.PoolFilled))); + events$.next(asTaskPollingCycleEvent(asOk(FillPoolResult.PoolFilled))); + events$.next(asTaskPollingCycleEvent(asOk(FillPoolResult.PoolFilled))); + events$.next(asTaskPollingCycleEvent(asOk(FillPoolResult.RanOutOfCapacity))); + events$.next(asTaskPollingCycleEvent(asOk(FillPoolResult.RanOutOfCapacity))); + events$.next(asTaskPollingCycleEvent(asOk(FillPoolResult.NoTasksClaimed))); + events$.next(asTaskPollingCycleEvent(asOk(FillPoolResult.NoTasksClaimed))); + }); + }); +}); + +function runAtMillisecondsAgo(ms: number): Date { + return new Date(Date.now() - ms); +} + +const mockTaskRunEvent = ( + overrides: Partial = {}, + timing: TaskTiming, + result: TaskRunResult = TaskRunResult.Success +) => { + const task = mockTaskInstance(overrides); + return asTaskRunEvent(task.id, asOk({ task, result }), timing); +}; + +const mockTaskInstance = (overrides: Partial = {}): ConcreteTaskInstance => ({ + id: uuid.v4(), + attempts: 0, + status: TaskStatus.Running, + version: '123', + runAt: new Date(), + scheduledAt: new Date(), + startedAt: new Date(), + retryAt: new Date(Date.now() + 5 * 60 * 1000), + state: {}, + taskType: 'alerting:test', + params: { + alertId: '1', + }, + ownerId: null, + ...overrides, +}); + +const getTaskManagerConfig = (overrides: unknown = {}) => configSchema.validate(overrides); diff --git a/x-pack/plugins/task_manager/server/monitoring/task_run_statistics.ts b/x-pack/plugins/task_manager/server/monitoring/task_run_statistics.ts new file mode 100644 index 00000000000000..6dd533177a86e4 --- /dev/null +++ b/x-pack/plugins/task_manager/server/monitoring/task_run_statistics.ts @@ -0,0 +1,224 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { combineLatest, Observable } from 'rxjs'; +import { filter, startWith, map } from 'rxjs/operators'; +import { JsonObject } from 'src/plugins/kibana_utils/common'; +import { mapValues } from 'lodash'; +import { AggregatedStatProvider, AggregatedStat } from './runtime_statistics_aggregator'; +import { TaskLifecycleEvent } from '../polling_lifecycle'; +import { + isTaskRunEvent, + isTaskPollingCycleEvent, + TaskRun, + ErroredTask, + RanTask, + TaskTiming, +} from '../task_events'; +import { isOk, Ok, unwrap } from '../lib/result_type'; +import { ConcreteTaskInstance } from '../task'; +import { TaskRunResult } from '../task_runner'; +import { FillPoolResult } from '../lib/fill_pool'; +import { + AveragedStat, + calculateRunningAverage, + calculateFrequency, + createRunningAveragedStat, + createMapOfRunningAveragedStats, +} from './task_run_calcultors'; +import { HealthStatus } from './monitoring_stats_stream'; +import { TaskPollingLifecycle } from '../polling_lifecycle'; +import { TaskExecutionFailureThreshold, TaskManagerConfig } from '../config'; + +interface FillPoolStat extends JsonObject { + last_successful_poll: string; + result_frequency_percent_as_number: FillPoolResult[]; +} + +interface ExecutionStat extends JsonObject { + duration: Record; + result_frequency_percent_as_number: Record; +} + +export interface TaskRunStat extends JsonObject { + drift: number[]; + execution: ExecutionStat; + polling: FillPoolStat | Omit; +} + +interface FillPoolRawStat extends JsonObject { + last_successful_poll: string; + result_frequency_percent_as_number: { + [FillPoolResult.NoTasksClaimed]: number; + [FillPoolResult.RanOutOfCapacity]: number; + [FillPoolResult.PoolFilled]: number; + }; +} + +interface ResultFrequency extends JsonObject { + [TaskRunResult.Success]: number; + [TaskRunResult.SuccessRescheduled]: number; + [TaskRunResult.RetryScheduled]: number; + [TaskRunResult.Failed]: number; +} + +type ResultFrequencySummary = ResultFrequency & { + status: HealthStatus; +}; + +export interface SummarizedTaskRunStat extends JsonObject { + drift: AveragedStat; + execution: { + duration: Record; + result_frequency_percent_as_number: Record; + }; + polling: FillPoolRawStat | Omit; +} + +export function createTaskRunAggregator( + taskPollingLifecycle: TaskPollingLifecycle, + runningAverageWindowSize: number +): AggregatedStatProvider { + const taskRunEventToStat = createTaskRunEventToStat(runningAverageWindowSize); + const taskRunEvents$: Observable> = taskPollingLifecycle.events.pipe( + filter((taskEvent: TaskLifecycleEvent) => isTaskRunEvent(taskEvent) && hasTiming(taskEvent)), + map((taskEvent: TaskLifecycleEvent) => { + const { task, result }: RanTask | ErroredTask = unwrap((taskEvent as TaskRun).event); + return taskRunEventToStat(task, taskEvent.timing!, result); + }) + ); + + const resultFrequencyQueue = createRunningAveragedStat(runningAverageWindowSize); + const taskPollingEvents$: Observable> = taskPollingLifecycle.events.pipe( + filter( + (taskEvent: TaskLifecycleEvent) => + isTaskPollingCycleEvent(taskEvent) && isOk(taskEvent.event) + ), + map((taskEvent: TaskLifecycleEvent) => { + return { + polling: { + last_successful_poll: new Date().toISOString(), + result_frequency_percent_as_number: resultFrequencyQueue( + ((taskEvent.event as unknown) as Ok).value + ), + }, + }; + }) + ); + + return combineLatest([ + taskRunEvents$.pipe( + startWith({ drift: [], execution: { duration: {}, result_frequency_percent_as_number: {} } }) + ), + taskPollingEvents$.pipe( + startWith({ + polling: { result_frequency_percent_as_number: [] }, + }) + ), + ]).pipe( + map(([taskRun, polling]: [Omit, Pick]) => { + return { + key: 'runtime', + value: { + ...taskRun, + ...polling, + }, + } as AggregatedStat; + }) + ); +} + +function hasTiming(taskEvent: TaskLifecycleEvent) { + return !!taskEvent?.timing; +} + +function createTaskRunEventToStat(runningAverageWindowSize: number) { + const driftQueue = createRunningAveragedStat(runningAverageWindowSize); + const taskRunDurationQueue = createMapOfRunningAveragedStats(runningAverageWindowSize); + const resultFrequencyQueue = createMapOfRunningAveragedStats( + runningAverageWindowSize + ); + return ( + task: ConcreteTaskInstance, + timing: TaskTiming, + result: TaskRunResult + ): Omit => ({ + drift: driftQueue(timing!.start - task.runAt.getTime()), + execution: { + duration: taskRunDurationQueue(task.taskType, timing!.stop - timing!.start), + result_frequency_percent_as_number: resultFrequencyQueue(task.taskType, result), + }, + }); +} + +const DEFAULT_TASK_RUN_FREQUENCIES = { + [TaskRunResult.Success]: 0, + [TaskRunResult.SuccessRescheduled]: 0, + [TaskRunResult.RetryScheduled]: 0, + [TaskRunResult.Failed]: 0, +}; +const DEFAULT_POLLING_FREQUENCIES = { + [FillPoolResult.NoTasksClaimed]: 0, + [FillPoolResult.RanOutOfCapacity]: 0, + [FillPoolResult.PoolFilled]: 0, +}; + +export function summarizeTaskRunStat( + { + // eslint-disable-next-line @typescript-eslint/naming-convention + polling: { last_successful_poll, result_frequency_percent_as_number: pollingResultFrequency }, + drift, + execution: { duration, result_frequency_percent_as_number: executionResultFrequency }, + }: TaskRunStat, + config: TaskManagerConfig +): { value: SummarizedTaskRunStat; status: HealthStatus } { + return { + value: { + polling: { + ...(last_successful_poll ? { last_successful_poll } : {}), + result_frequency_percent_as_number: { + ...DEFAULT_POLLING_FREQUENCIES, + ...calculateFrequency(pollingResultFrequency as FillPoolResult[]), + }, + }, + drift: calculateRunningAverage(drift), + execution: { + duration: mapValues(duration, (typedDurations) => calculateRunningAverage(typedDurations)), + result_frequency_percent_as_number: mapValues( + executionResultFrequency, + (typedResultFrequencies, taskType) => + summarizeTaskExecutionResultFrequencyStat( + { + ...DEFAULT_TASK_RUN_FREQUENCIES, + ...calculateFrequency(typedResultFrequencies), + }, + config.monitored_task_execution_thresholds.custom[taskType] ?? + config.monitored_task_execution_thresholds.default + ) + ), + }, + }, + status: HealthStatus.OK, + }; +} + +function summarizeTaskExecutionResultFrequencyStat( + resultFrequencySummary: ResultFrequency, + executionErrorThreshold: TaskExecutionFailureThreshold +): ResultFrequencySummary { + return { + ...resultFrequencySummary, + status: + resultFrequencySummary.Failed > executionErrorThreshold.warn_threshold + ? resultFrequencySummary.Failed > executionErrorThreshold.error_threshold + ? HealthStatus.Error + : HealthStatus.Warning + : HealthStatus.OK, + }; +} diff --git a/x-pack/plugins/task_manager/server/monitoring/workload_statistics.test.ts b/x-pack/plugins/task_manager/server/monitoring/workload_statistics.test.ts new file mode 100644 index 00000000000000..d9af3307e75cbb --- /dev/null +++ b/x-pack/plugins/task_manager/server/monitoring/workload_statistics.test.ts @@ -0,0 +1,773 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { first, take, bufferCount } from 'rxjs/operators'; +import { loggingSystemMock } from '../../../../../src/core/server/mocks'; +import { + WorkloadAggregation, + createWorkloadAggregator, + padBuckets, + estimateRecurringTaskScheduling, +} from './workload_statistics'; +import { ConcreteTaskInstance } from '../task'; +import { ESSearchResponse } from '../../../apm/typings/elasticsearch'; +import { AggregationResultOf } from '../../../apm/typings/elasticsearch/aggregations'; +import { times } from 'lodash'; +import { taskStoreMock } from '../task_store.mock'; + +type MockESResult = ESSearchResponse< + ConcreteTaskInstance, + { + body: WorkloadAggregation; + } +>; + +describe('Workload Statistics Aggregator', () => { + test('queries the Task Store at a fixed interval for the current workload', async () => { + const taskStore = taskStoreMock.create({}); + taskStore.aggregate.mockResolvedValue({ + hits: { + hits: [], + max_score: 0, + total: { value: 0, relation: 'eq' }, + }, + took: 1, + timed_out: false, + _shards: { + total: 1, + successful: 1, + skipped: 1, + failed: 0, + }, + aggregations: { + taskType: { + buckets: [], + }, + schedule: { + buckets: [], + }, + idleTasks: { + doc_count: 0, + overdue: { + doc_count: 0, + }, + scheduleDensity: { + buckets: [ + { + key: '2020-10-02T15:18:37.274Z-2020-10-02T15:19:36.274Z', + from: 1.601651917274e12, + from_as_string: '2020-10-02T15:18:37.274Z', + to: 1.601651976274e12, + to_as_string: '2020-10-02T15:19:36.274Z', + doc_count: 0, + histogram: { + buckets: [], + }, + }, + ], + }, + }, + }, + } as MockESResult); + + const workloadAggregator = createWorkloadAggregator( + taskStore, + 10, + 3000, + loggingSystemMock.create().get() + ); + + return new Promise((resolve) => { + workloadAggregator.pipe(first()).subscribe(() => { + expect(taskStore.aggregate).toHaveBeenCalledWith({ + aggs: { + taskType: { + terms: { field: 'task.taskType' }, + aggs: { + status: { + terms: { field: 'task.status' }, + }, + }, + }, + schedule: { + terms: { + field: 'task.schedule.interval', + }, + }, + idleTasks: { + filter: { + term: { 'task.status': 'idle' }, + }, + aggs: { + scheduleDensity: { + range: { + field: 'task.runAt', + ranges: [{ from: 'now', to: 'now+1m' }], + }, + aggs: { + histogram: { + date_histogram: { + field: 'task.runAt', + fixed_interval: '3s', + }, + aggs: { + interval: { + terms: { + field: 'task.schedule.interval', + }, + }, + }, + }, + }, + }, + overdue: { + filter: { + range: { + 'task.runAt': { lt: 'now' }, + }, + }, + }, + }, + }, + }, + }); + resolve(); + }); + }); + }); + + const mockAggregatedResult: () => MockESResult = () => + ({ + hits: { + hits: [], + max_score: 0, + total: { value: 4, relation: 'eq' }, + }, + took: 1, + timed_out: false, + _shards: { + total: 1, + successful: 1, + skipped: 1, + failed: 0, + }, + aggregations: { + schedule: { + buckets: [ + { + key: '3600s', + doc_count: 1, + }, + { + key: '60s', + doc_count: 1, + }, + { + key: '720m', + doc_count: 1, + }, + ], + }, + taskType: { + buckets: [ + { + key: 'actions_telemetry', + doc_count: 2, + status: { + buckets: [ + { + key: 'idle', + doc_count: 2, + }, + ], + }, + }, + { + key: 'alerting_telemetry', + doc_count: 1, + status: { + buckets: [ + { + key: 'idle', + doc_count: 1, + }, + ], + }, + }, + { + key: 'session_cleanup', + doc_count: 1, + status: { + buckets: [ + { + key: 'idle', + doc_count: 1, + }, + ], + }, + }, + ], + }, + idleTasks: { + doc_count: 13, + overdue: { + doc_count: 6, + }, + scheduleDensity: { + buckets: [ + mockHistogram(0, 7 * 3000 + 500, 60 * 1000, 3000, [2, 2, 5, 0, 0, 0, 0, 0, 0, 1]), + ], + }, + }, + }, + } as MockESResult); + + test('returns a summary of the workload by task type', async () => { + const taskStore = taskStoreMock.create({}); + taskStore.aggregate.mockResolvedValue(mockAggregatedResult()); + + const workloadAggregator = createWorkloadAggregator( + taskStore, + 10, + 3000, + loggingSystemMock.create().get() + ); + + return new Promise((resolve) => { + workloadAggregator.pipe(first()).subscribe((result) => { + expect(result.key).toEqual('workload'); + expect(result.value).toMatchObject({ + count: 4, + task_types: { + actions_telemetry: { count: 2, status: { idle: 2 } }, + alerting_telemetry: { count: 1, status: { idle: 1 } }, + session_cleanup: { count: 1, status: { idle: 1 } }, + }, + }); + resolve(); + }); + }); + }); + + test('returns a count of the overdue workload', async () => { + const taskStore = taskStoreMock.create({}); + taskStore.aggregate.mockResolvedValue(mockAggregatedResult()); + + const workloadAggregator = createWorkloadAggregator( + taskStore, + 10, + 3000, + loggingSystemMock.create().get() + ); + + return new Promise((resolve) => { + workloadAggregator.pipe(first()).subscribe((result) => { + expect(result.key).toEqual('workload'); + expect(result.value).toMatchObject({ + overdue: 6, + }); + resolve(); + }); + }); + }); + + test('returns a histogram of the upcoming workload for the upcoming minute when refresh rate is high', async () => { + const taskStore = taskStoreMock.create({}); + taskStore.aggregate.mockResolvedValue(mockAggregatedResult()); + + const workloadAggregator = createWorkloadAggregator( + taskStore, + 10, + 3000, + loggingSystemMock.create().get() + ); + + return new Promise((resolve) => { + workloadAggregator.pipe(first()).subscribe((result) => { + expect(result.key).toEqual('workload'); + expect(result.value).toMatchObject({ + // we have intervals every 3s, so we aggregate buckets 3s apart + // in this mock, Elasticsearch found tasks scheduled in 21 (8th bucket), 24, 27 and 48s seconds from now + // 0, 3, 6, 9, 12, 15, 18, 21, 24, 27, 30, 33, 36, 39, 42, 45, 48, 51, 54, 57 + // [0, 0, 0, 0, 0, 0, 0, 2, 2, 5, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0 ] + // Above you see each bucket and the number of scheduled tasks we expect to have in them + estimated_schedule_density: [0, 0, 0, 0, 0, 0, 0, 2, 2, 5, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0], + }); + resolve(); + }); + }); + }); + + test('returns a histogram of the upcoming workload for twice refresh rate when rate is low', async () => { + const taskStore = taskStoreMock.create({}); + taskStore.aggregate.mockResolvedValue(mockAggregatedResult()); + + const workloadAggregator = createWorkloadAggregator( + taskStore, + 60 * 1000, + 3000, + loggingSystemMock.create().get() + ); + + return new Promise((resolve) => { + workloadAggregator.pipe(first()).subscribe(() => { + expect(taskStore.aggregate.mock.calls[0][0]).toMatchObject({ + aggs: { + idleTasks: { + aggs: { + scheduleDensity: { + range: { + field: 'task.runAt', + ranges: [ + { + from: 'now', + to: 'now+2m', + }, + ], + }, + }, + }, + }, + }, + }); + resolve(); + }); + }); + }); + + test('returns a histogram of the upcoming workload maxed out at 50 buckets when rate is too low', async () => { + const taskStore = taskStoreMock.create({}); + taskStore.aggregate.mockResolvedValue(mockAggregatedResult()); + + const workloadAggregator = createWorkloadAggregator( + taskStore, + 15 * 60 * 1000, + 3000, + loggingSystemMock.create().get() + ); + + return new Promise((resolve) => { + workloadAggregator.pipe(first()).subscribe((result) => { + expect(taskStore.aggregate.mock.calls[0][0]).toMatchObject({ + aggs: { + idleTasks: { + aggs: { + scheduleDensity: { + range: { + field: 'task.runAt', + ranges: [ + { + from: 'now', + // 50 buckets of 3s = 50 * 3 = 150s + to: 'now+150s', + }, + ], + }, + }, + }, + }, + }, + }); + resolve(); + }); + }); + }); + + test('recovers from errors fetching the workload', async () => { + const taskStore = taskStoreMock.create({}); + taskStore.aggregate + .mockResolvedValueOnce( + setTaskTypeCount(mockAggregatedResult(), 'alerting_telemetry', { + idle: 2, + }) + ) + .mockRejectedValueOnce(new Error('Elasticsearch has gone poof')) + .mockResolvedValueOnce( + setTaskTypeCount(mockAggregatedResult(), 'alerting_telemetry', { + idle: 1, + failed: 1, + }) + ); + const logger = loggingSystemMock.create().get(); + const workloadAggregator = createWorkloadAggregator(taskStore, 10, 3000, logger); + + return new Promise((resolve, reject) => { + workloadAggregator.pipe(take(2), bufferCount(2)).subscribe((results) => { + expect(results[0].key).toEqual('workload'); + expect(results[0].value).toMatchObject({ + count: 5, + task_types: { + actions_telemetry: { count: 2, status: { idle: 2 } }, + alerting_telemetry: { count: 2, status: { idle: 2 } }, + session_cleanup: { count: 1, status: { idle: 1 } }, + }, + }); + expect(results[1].key).toEqual('workload'); + expect(results[1].value).toMatchObject({ + count: 5, + task_types: { + actions_telemetry: { count: 2, status: { idle: 2 } }, + alerting_telemetry: { count: 2, status: { idle: 1, failed: 1 } }, + session_cleanup: { count: 1, status: { idle: 1 } }, + }, + }); + resolve(); + }, reject); + }); + }); +}); + +describe('estimateRecurringTaskScheduling', () => { + test('flattens out buckets with non recurring tasks', () => { + const now = Date.now(); + const schedule = times(10, (index) => ({ + key: index * 3000 + now, + nonRecurring: index, + })); + expect(estimateRecurringTaskScheduling(schedule, 3000)).toEqual([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); + }); + + test('estimates the buckets that recurring tasks might repeat in when recurring task interval equals the interval', () => { + const now = Date.now(); + const schedule: Array<{ + key: number; + nonRecurring: number; + recurring?: Array<[number, string]>; + }> = times(10, (index) => ({ + key: index * 3000 + now, + nonRecurring: 0, + })); + + schedule[0].nonRecurring = 1; + schedule[1].nonRecurring = 1; + schedule[4].recurring = [[1, '3s']]; + + expect(estimateRecurringTaskScheduling(schedule, 3000)).toEqual([1, 1, 0, 0, 1, 1, 1, 1, 1, 1]); + }); + + test('estimates the buckets that recurring tasks might repeat in when recurring task interval is larger than the interval', () => { + const now = Date.now(); + const schedule: Array<{ + key: number; + nonRecurring: number; + recurring?: Array<[number, string]>; + }> = times(10, (index) => ({ + key: index * 3000 + now, + nonRecurring: 0, + })); + + schedule[0].nonRecurring = 1; + schedule[1].nonRecurring = 1; + schedule[4].recurring = [[1, '6s']]; + + expect(estimateRecurringTaskScheduling(schedule, 3000)).toEqual([1, 1, 0, 0, 1, 0, 1, 0, 1, 0]); + }); + + test('estimates the buckets that recurring tasks might repeat in when recurring task interval doesnt divide by interval', () => { + const now = Date.now(); + const schedule: Array<{ + key: number; + nonRecurring: number; + recurring?: Array<[number, string]>; + }> = times(10, (index) => ({ + key: index * 3000 + now, + nonRecurring: 0, + })); + + schedule[0].nonRecurring = 1; + schedule[1].nonRecurring = 1; + schedule[4].recurring = [[1, '5s']]; + + expect(estimateRecurringTaskScheduling(schedule, 3000)).toEqual([1, 1, 0, 0, 1, 0, 1, 0, 1, 0]); + }); + + test('estimates the buckets that recurring tasks might repeat in when recurring tasks overlap', () => { + const now = Date.now(); + const schedule: Array<{ + key: number; + nonRecurring: number; + recurring?: Array<[number, string]>; + }> = times(20, (index) => ({ + key: index * 3000 + now, + nonRecurring: 0, + })); + + schedule[0].nonRecurring = 1; + schedule[1].nonRecurring = 1; + schedule[3].recurring = [[1, '3s']]; + schedule[4].recurring = [ + [2, '6s'], + [1, '8s'], + ]; + schedule[5].recurring = [[1, '5s']]; + schedule[6].nonRecurring = 3; + + expect(estimateRecurringTaskScheduling(schedule, 3000)).toEqual([ + 1, + 1, + 0, + 1, + 4, + 2, + 6, + 3, + 3, + 2, + 4, + 2, + 3, + 3, + 3, + 2, + 4, + 2, + 3, + 3, + ]); + }); +}); + +describe('padBuckets', () => { + test('returns zeroed out bucklets when there are no buckets in the histogram', async () => { + expect( + padBuckets(10, 3000, { + key: '2020-10-02T19:47:28.128Z-2020-10-02T19:48:28.128Z', + from: 1601668048128, + from_as_string: '2020-10-02T19:47:28.128Z', + to: 1601668108128, + to_as_string: '2020-10-02T19:48:28.128Z', + doc_count: 0, + histogram: { + buckets: [], + }, + }) + ).toEqual([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + }); + + test('pads buckets with zeros to fill out the entire period of time after detected buckets', async () => { + expect( + padBuckets(10, 3000, { + key: '2020-10-02T19:47:28.128Z-2020-10-02T19:48:28.128Z', + from: 1601668046000, + from_as_string: '2020-10-02T19:47:26.000Z', + to: 1601668076000, + to_as_string: '2020-10-02T19:47:56.000Z', + doc_count: 3, + histogram: { + buckets: [ + { + key_as_string: '2020-10-02T19:47:27.000Z', + key: 1601668047000, + doc_count: 1, + interval: { + sum_other_doc_count: 0, + buckets: [], + }, + }, + { + key_as_string: '2020-10-02T19:47:30.000Z', + key: 1601668050000, + doc_count: 1, + interval: { + sum_other_doc_count: 0, + buckets: [], + }, + }, + { + key_as_string: '2020-10-02T19:47:33.000Z', + key: 1601668053000, + doc_count: 0, + interval: { + sum_other_doc_count: 0, + buckets: [], + }, + }, + { + key_as_string: '2020-10-02T19:47:36.000Z', + key: 1601668056000, + doc_count: 0, + interval: { + sum_other_doc_count: 0, + buckets: [], + }, + }, + { + key_as_string: '2020-10-02T19:47:39.000Z', + key: 1601668059000, + doc_count: 0, + interval: { + sum_other_doc_count: 0, + buckets: [], + }, + }, + { + key_as_string: '2020-10-02T19:47:42.000Z', + key: 1601668062000, + doc_count: 1, + interval: { + sum_other_doc_count: 0, + buckets: [], + }, + }, + ], + }, + }) + ).toEqual([1, 1, 0, 0, 0, 1, 0, 0, 0, 0]); + }); + + test('pads buckets with zeros to fill out the entire period of time before detected buckets', async () => { + expect( + padBuckets(10, 3000, { + key: '2020-10-02T20:39:45.793Z-2020-10-02T20:40:14.793Z', + from: 1601671183000, + from_as_string: '2020-10-02T20:39:43.000Z', + to: 1601671213000, + to_as_string: '2020-10-02T20:40:13.000Z', + doc_count: 2, + histogram: { + buckets: [ + { + key_as_string: '2020-10-02T20:40:09.000Z', + key: 1601671209000, + doc_count: 1, + interval: { buckets: [] }, + }, + { + key_as_string: '2020-10-02T20:40:12.000Z', + key: 1601671212000, + doc_count: 1, + interval: { buckets: [] }, + }, + ], + }, + }) + ).toEqual([0, 0, 0, 0, 0, 0, 0, 0, 1, 1]); + }); + + test('pads buckets with zeros to fill out the entire period surounding the detected buckets', async () => { + expect( + padBuckets(20, 3000, { + key: '2020-10-02T20:39:45.793Z-2020-10-02T20:40:14.793Z', + from: 1601671185793, + from_as_string: '2020-10-02T20:39:45.793Z', + to: 1601671245793, + to_as_string: '2020-10-02T20:40:45.793Z', + doc_count: 2, + histogram: { + buckets: [ + { + key_as_string: '2020-10-02T20:40:09.000Z', + key: 1601671209000, + doc_count: 1, + interval: { buckets: [] }, + }, + { + key_as_string: '2020-10-02T20:40:12.000Z', + key: 1601671212000, + doc_count: 1, + interval: { buckets: [] }, + }, + ], + }, + }) + ).toEqual([0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + }); +}); + +function setTaskTypeCount( + { aggregations }: MockESResult, + taskType: string, + status: Record +) { + const taskTypes = aggregations!.taskType as AggregationResultOf< + WorkloadAggregation['aggs']['taskType'], + {} + >; + const buckets = [ + ...taskTypes.buckets.filter(({ key }) => key !== taskType), + { + key: taskType, + doc_count: Object.values(status).reduce((sum, count) => sum + count, 0), + status: { + sum_other_doc_count: 0, + buckets: Object.entries(status).map(([key, count]) => ({ + key, + doc_count: count, + })), + }, + }, + ]; + return ({ + hits: { + total: { value: buckets.reduce((sum, bucket) => sum + bucket.doc_count, 0) }, + }, + aggregations: { + ...aggregations, + taskType: { + sum_other_doc_count: 0, + buckets, + }, + }, + } as {}) as MockESResult; +} + +/** * + * This creates a mock histogram as returned by Elasticsearch + * + * @param from lower bound of query + * @param findFrom the timestamp (key) of the first bucket returned + * @param to upper bound of query + * @param interval the duration that each bucket coresponds to + * @param foundBuckets the buckets identified by ES, any buckets missing before or after which + * are still in the date range are assumed to have 0 results, ES only returns 0 for + * buckets that sit in between buckets which do have results + */ +function mockHistogram( + from: number, + findFrom: number, + to: number, + interval: number, + foundBuckets: Array +) { + const now = Date.now(); + const fromDate = new Date(now + from); + const toDate = new Date(now + to); + return { + key: `${fromDate.toISOString()}-${toDate.toISOString()}`, + from: now + from, + from_as_string: fromDate.toISOString(), + to: now + to, + to_as_string: toDate.toISOString(), + doc_count: foundBuckets.reduce((sum: number, count) => sum + (count ?? 0), 0), + histogram: { + buckets: foundBuckets.reduce( + (histogramBuckets, count, index) => { + if (typeof count === 'number') { + const key = new Date(now + findFrom + index * interval); + histogramBuckets.push({ + key_as_string: key.toISOString(), + key: key.getTime(), + doc_count: count, + interval: { buckets: [] }, + }); + } + return histogramBuckets; + }, + [] as Array<{ + key_as_string: string; + key: number; + doc_count: number; + interval: { + buckets: Array<{ + key: string; + doc_count: number; + }>; + }; + }> + ), + }, + }; +} diff --git a/x-pack/plugins/task_manager/server/monitoring/workload_statistics.ts b/x-pack/plugins/task_manager/server/monitoring/workload_statistics.ts new file mode 100644 index 00000000000000..fe70f24684ad94 --- /dev/null +++ b/x-pack/plugins/task_manager/server/monitoring/workload_statistics.ts @@ -0,0 +1,344 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { timer } from 'rxjs'; +import { mergeMap, map, catchError } from 'rxjs/operators'; +import { Logger } from 'src/core/server'; +import { JsonObject } from 'src/plugins/kibana_utils/common'; +import { keyBy, mapValues } from 'lodash'; +import { AggregatedStatProvider } from './runtime_statistics_aggregator'; +import { parseIntervalAsSecond, asInterval, parseIntervalAsMillisecond } from '../lib/intervals'; +import { AggregationResultOf } from '../../../apm/typings/elasticsearch/aggregations'; +import { HealthStatus } from './monitoring_stats_stream'; +import { TaskStore } from '../task_store'; + +interface StatusStat extends JsonObject { + [status: string]: number; +} +interface TaskTypeStat extends JsonObject { + [taskType: string]: { + count: number; + status: StatusStat; + }; +} + +export interface WorkloadStat extends JsonObject { + count: number; + task_types: TaskTypeStat; + schedule: Array<[string, number]>; + overdue: number; + estimated_schedule_density: number[]; +} + +export interface WorkloadAggregation { + aggs: { + taskType: { + terms: { field: string }; + aggs: { + status: { + terms: { field: string }; + }; + }; + }; + schedule: { + terms: { field: string }; + }; + idleTasks: { + filter: { + term: { 'task.status': string }; + }; + aggs: { + scheduleDensity: { + range: { + field: string; + ranges: [{ from: string; to: string }]; + }; + aggs: { + histogram: { + date_histogram: { + field: string; + fixed_interval: string; + }; + aggs: { + interval: { + terms: { field: string }; + }; + }; + }; + }; + }; + overdue: { + filter: { + range: { + 'task.runAt': { lt: string }; + }; + }; + }; + }; + }; + }; +} + +// The type of a bucket in the scheduleDensity range aggregation +type ScheduleDensityResult = AggregationResultOf< + WorkloadAggregation['aggs']['idleTasks']['aggs']['scheduleDensity'], + {} +>['buckets'][0]; +type ScheduledIntervals = ScheduleDensityResult['histogram']['buckets'][0]; + +// Set an upper bound just in case a customer sets a really high refresh rate +const MAX_SHCEDULE_DENSITY_BUCKETS = 50; + +export function createWorkloadAggregator( + taskStore: TaskStore, + refreshInterval: number, + pollInterval: number, + logger: Logger +): AggregatedStatProvider { + // calculate scheduleDensity going two refreshIntervals or 1 minute into into the future + // (the longer of the two) + const scheduleDensityBuckets = Math.min( + Math.max(Math.round(60000 / pollInterval), Math.round((refreshInterval * 2) / pollInterval)), + MAX_SHCEDULE_DENSITY_BUCKETS + ); + + return timer(0, refreshInterval).pipe( + mergeMap(() => + taskStore.aggregate({ + aggs: { + taskType: { + terms: { field: 'task.taskType' }, + aggs: { + status: { + terms: { field: 'task.status' }, + }, + }, + }, + schedule: { + terms: { field: 'task.schedule.interval' }, + }, + idleTasks: { + filter: { + term: { 'task.status': 'idle' }, + }, + aggs: { + scheduleDensity: { + // create a window of upcoming tasks + range: { + field: 'task.runAt', + ranges: [ + { + from: `now`, + to: `now+${asInterval(scheduleDensityBuckets * pollInterval)}`, + }, + ], + }, + aggs: { + // create histogram of scheduling in the window, with each bucket being a polling interval + histogram: { + date_histogram: { + field: 'task.runAt', + fixed_interval: asInterval(pollInterval), + }, + // break down each bucket in the historgram by schedule + aggs: { + interval: { + terms: { field: 'task.schedule.interval' }, + }, + }, + }, + }, + }, + overdue: { + filter: { + range: { + 'task.runAt': { lt: 'now' }, + }, + }, + }, + }, + }, + }, + }) + ), + map((result) => { + const { + aggregations, + hits: { + total: { value: count }, + }, + } = result; + + if ( + !( + aggregations?.taskType && + aggregations?.schedule && + aggregations?.idleTasks?.overdue && + aggregations?.idleTasks?.scheduleDensity + ) + ) { + throw new Error(`Invalid workload: ${JSON.stringify(result)}`); + } + + const taskTypes = aggregations.taskType.buckets; + const schedules = aggregations.schedule.buckets; + + const { + overdue: { doc_count: overdue }, + scheduleDensity: { buckets: [scheduleDensity] = [] } = {}, + } = aggregations.idleTasks; + + const summary: WorkloadStat = { + count, + task_types: mapValues(keyBy(taskTypes, 'key'), ({ doc_count: docCount, status }) => { + return { + count: docCount, + status: mapValues(keyBy(status.buckets, 'key'), 'doc_count'), + }; + }), + schedule: schedules + .sort( + (scheduleLeft, scheduleRight) => + parseIntervalAsSecond(scheduleLeft.key as string) - + parseIntervalAsSecond(scheduleRight.key as string) + ) + .map((schedule) => [schedule.key as string, schedule.doc_count]), + overdue, + estimated_schedule_density: padBuckets( + scheduleDensityBuckets, + pollInterval, + scheduleDensity + ), + }; + return { + key: 'workload', + value: summary, + }; + }), + catchError((ex: Error, caught) => { + logger.error(`[WorkloadAggregator]: ${ex}`); + // continue to pull values from the same observable + return caught; + }) + ); +} + +interface IntervalTaskCountTouple { + nonRecurring?: number; + recurring?: Array<[number, string]>; + key: number; +} + +export function padBuckets( + scheduleDensityBuckets: number, + pollInterval: number, + scheduleDensity: ScheduleDensityResult +): number[] { + if (scheduleDensity.from && scheduleDensity.to && scheduleDensity.histogram?.buckets?.length) { + const { histogram, from, to } = scheduleDensity; + const firstBucket = histogram.buckets[0].key; + const lastBucket = histogram.buckets[histogram.buckets.length - 1].key; + + const bucketsToPadBeforeFirstBucket = calculateBucketsBetween(firstBucket, from, pollInterval); + const bucketsToPadAfterLast = calculateBucketsBetween( + lastBucket + pollInterval, + to, + pollInterval + ); + + return estimateRecurringTaskScheduling( + [ + ...bucketsToPadBeforeFirstBucket, + ...histogram.buckets.map(countByIntervalInBucket), + ...bucketsToPadAfterLast, + ], + pollInterval + ); + } + return new Array(scheduleDensityBuckets).fill(0); +} + +function countByIntervalInBucket(bucket: ScheduledIntervals): IntervalTaskCountTouple { + if (bucket.doc_count === 0) { + return { nonRecurring: 0, key: bucket.key }; + } + const recurring: Array<[number, string]> = []; + let nonRecurring = bucket.doc_count; + for (const intervalBucket of bucket.interval.buckets) { + recurring.push([intervalBucket.doc_count, intervalBucket.key as string]); + nonRecurring -= intervalBucket.doc_count; + } + + return { nonRecurring, recurring, key: bucket.key }; +} + +function calculateBucketsBetween( + from: number, + to: number, + interval: number, + bucketInterval: number = interval +): Array<{ key: number }> { + const calcForwardInTime = from < to; + + // as task interval might not divide by the pollInterval (aka the bucket interval) + // we have to adjust for the "drift" that occurs when estimating when the next + // bucket the task might actually get scheduled in + const actualInterval = Math.ceil(interval / bucketInterval) * bucketInterval; + + const buckets: Array<{ key: number }> = []; + const toBound = calcForwardInTime ? to : -(to + actualInterval); + let fromBound = calcForwardInTime ? from : -from; + + while (fromBound < toBound) { + buckets.push({ key: fromBound }); + fromBound += actualInterval; + } + + return calcForwardInTime + ? buckets + : buckets.reverse().map((bucket) => { + bucket.key = Math.abs(bucket.key); + return bucket; + }); +} + +export function estimateRecurringTaskScheduling( + scheduleDensity: IntervalTaskCountTouple[], + pollInterval: number +) { + const lastKey = scheduleDensity[scheduleDensity.length - 1].key; + + return scheduleDensity.map((bucket, currentBucketIndex) => { + for (const [count, interval] of bucket.recurring ?? []) { + for (const recurrance of calculateBucketsBetween( + bucket.key, + // `calculateBucketsBetween` uses the `to` as a non-inclusive upper bound + // but lastKey is a bucket we wish to include + lastKey + pollInterval, + parseIntervalAsMillisecond(interval), + pollInterval + )) { + const recurranceBucketIndex = + currentBucketIndex + Math.ceil((recurrance.key - bucket.key) / pollInterval); + + if (recurranceBucketIndex < scheduleDensity.length) { + scheduleDensity[recurranceBucketIndex].nonRecurring = + count + (scheduleDensity[recurranceBucketIndex].nonRecurring ?? 0); + } + } + } + return bucket.nonRecurring ?? 0; + }); +} + +export function summarizeWorkloadStat( + workloadStats: WorkloadStat +): { value: WorkloadStat; status: HealthStatus } { + return { + value: workloadStats, + status: HealthStatus.OK, + }; +} diff --git a/x-pack/plugins/task_manager/server/plugin.test.ts b/x-pack/plugins/task_manager/server/plugin.test.ts index 50e7e9a7aa1971..8388468164a4fd 100644 --- a/x-pack/plugins/task_manager/server/plugin.test.ts +++ b/x-pack/plugins/task_manager/server/plugin.test.ts @@ -19,6 +19,16 @@ describe('TaskManagerPlugin', () => { poll_interval: 3000, max_poll_inactivity_cycles: 10, request_capacity: 1000, + monitored_aggregated_stats_refresh_rate: 5000, + monitored_stats_required_freshness: 5000, + monitored_stats_running_average_window: 50, + monitored_task_execution_thresholds: { + default: { + error_threshold: 90, + warn_threshold: 80, + }, + custom: {}, + }, }); pluginInitializerContext.env.instanceUuid = ''; @@ -38,6 +48,16 @@ describe('TaskManagerPlugin', () => { poll_interval: 3000, max_poll_inactivity_cycles: 10, request_capacity: 1000, + monitored_aggregated_stats_refresh_rate: 5000, + monitored_stats_required_freshness: 5000, + monitored_stats_running_average_window: 50, + monitored_task_execution_thresholds: { + default: { + error_threshold: 90, + warn_threshold: 80, + }, + custom: {}, + }, }); const taskManagerPlugin = new TaskManagerPlugin(pluginInitializerContext); diff --git a/x-pack/plugins/task_manager/server/plugin.ts b/x-pack/plugins/task_manager/server/plugin.ts index 0381698e6fb770..0e7abb817490ae 100644 --- a/x-pack/plugins/task_manager/server/plugin.ts +++ b/x-pack/plugins/task_manager/server/plugin.ts @@ -4,7 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ import { PluginInitializerContext, Plugin, CoreSetup, Logger, CoreStart } from 'src/core/server'; -import { first } from 'rxjs/operators'; +import { combineLatest, Subject } from 'rxjs'; +import { first, map } from 'rxjs/operators'; import { TaskDefinition } from './task'; import { TaskPollingLifecycle } from './polling_lifecycle'; import { TaskManagerConfig } from './config'; @@ -14,6 +15,8 @@ import { TaskTypeDictionary } from './task_type_dictionary'; import { FetchResult, SearchOpts, TaskStore } from './task_store'; import { createManagedConfiguration } from './lib/create_managed_configuration'; import { TaskScheduling } from './task_scheduling'; +import { healthRoute } from './routes'; +import { createMonitoringStats, MonitoringStats } from './monitoring'; export type TaskManagerSetupContract = { addMiddleware: (middleware: Middleware) => void } & Pick< TaskTypeDictionary, @@ -34,6 +37,7 @@ export class TaskManagerPlugin private logger: Logger; private definitions: TaskTypeDictionary; private middleware: Middleware = createInitialMiddleware(); + private monitoringStats$ = new Subject(); constructor(private readonly initContext: PluginInitializerContext) { this.initContext = initContext; @@ -41,13 +45,13 @@ export class TaskManagerPlugin this.definitions = new TaskTypeDictionary(this.logger); } - public async setup({ savedObjects }: CoreSetup): Promise { + public async setup(core: CoreSetup): Promise { this.config = await this.initContext.config .create() .pipe(first()) .toPromise(); - setupSavedObjects(savedObjects, this.config); + setupSavedObjects(core.savedObjects, this.config); this.taskManagerId = this.initContext.env.instanceUuid; if (!this.taskManagerId) { @@ -59,6 +63,26 @@ export class TaskManagerPlugin this.logger.info(`TaskManager is identified by the Kibana UUID: ${this.taskManagerId}`); } + // Routes + const router = core.http.createRouter(); + const serviceStatus$ = healthRoute( + router, + this.monitoringStats$, + this.logger, + this.taskManagerId, + this.config! + ); + + core.getStartServices().then(async () => { + core.status.set( + combineLatest([core.status.derivedStatus$, serviceStatus$]).pipe( + map(([derivedStatus, serviceStatus]) => + serviceStatus.level > derivedStatus.level ? serviceStatus : derivedStatus + ) + ) + ); + }); + return { addMiddleware: (middleware: Middleware) => { this.assertStillInSetup('add Middleware'); @@ -84,7 +108,7 @@ export class TaskManagerPlugin taskManagerId: `kibana:${this.taskManagerId!}`, }); - const { maxWorkersConfiguration$, pollIntervalConfiguration$ } = createManagedConfiguration({ + const managedConfiguration = createManagedConfiguration({ logger: this.logger, errors$: taskStore.errors$, startingMaxWorkers: this.config!.max_workers, @@ -97,11 +121,18 @@ export class TaskManagerPlugin logger: this.logger, taskStore, middleware: this.middleware, - maxWorkersConfiguration$, - pollIntervalConfiguration$, + ...managedConfiguration, }); this.taskPollingLifecycle = taskPollingLifecycle; + createMonitoringStats( + taskPollingLifecycle, + taskStore, + this.config!, + managedConfiguration, + this.logger + ).subscribe((stat) => this.monitoringStats$.next(stat)); + const taskScheduling = new TaskScheduling({ logger: this.logger, taskStore, diff --git a/x-pack/plugins/task_manager/server/polling/task_poller.test.ts b/x-pack/plugins/task_manager/server/polling/task_poller.test.ts index 956c8b05f38607..f5f1667312d797 100644 --- a/x-pack/plugins/task_manager/server/polling/task_poller.test.ts +++ b/x-pack/plugins/task_manager/server/polling/task_poller.test.ts @@ -9,7 +9,8 @@ import { Subject, of, BehaviorSubject } from 'rxjs'; import { Option, none, some } from 'fp-ts/lib/Option'; import { createTaskPoller, PollingError, PollingErrorType } from './task_poller'; import { fakeSchedulers } from 'rxjs-marbles/jest'; -import { sleep, resolvable, Resolvable, mockLogger } from '../test_utils'; +import { sleep, resolvable, Resolvable } from '../test_utils'; +import { loggingSystemMock } from '../../../../../src/core/server/mocks'; import { asOk, asErr } from '../lib/result_type'; describe('TaskPoller', () => { @@ -24,7 +25,7 @@ describe('TaskPoller', () => { const work = jest.fn(async () => true); createTaskPoller({ - logger: mockLogger(), + logger: loggingSystemMock.create().get(), pollInterval$: of(pollInterval), bufferCapacity, getCapacity: () => 1, @@ -59,7 +60,7 @@ describe('TaskPoller', () => { const work = jest.fn(async () => true); createTaskPoller({ - logger: mockLogger(), + logger: loggingSystemMock.create().get(), pollInterval$, bufferCapacity, getCapacity: () => 1, @@ -101,7 +102,7 @@ describe('TaskPoller', () => { let hasCapacity = true; createTaskPoller({ - logger: mockLogger(), + logger: loggingSystemMock.create().get(), pollInterval$: of(pollInterval), bufferCapacity, work, @@ -160,7 +161,7 @@ describe('TaskPoller', () => { const work = jest.fn(async () => true); const pollRequests$ = new Subject>(); createTaskPoller({ - logger: mockLogger(), + logger: loggingSystemMock.create().get(), pollInterval$: of(pollInterval), bufferCapacity, work, @@ -206,7 +207,7 @@ describe('TaskPoller', () => { const work = jest.fn(async () => true); const pollRequests$ = new Subject>(); createTaskPoller({ - logger: mockLogger(), + logger: loggingSystemMock.create().get(), pollInterval$: of(pollInterval), bufferCapacity, work, @@ -251,7 +252,7 @@ describe('TaskPoller', () => { const work = jest.fn(async () => true); const pollRequests$ = new Subject>(); createTaskPoller({ - logger: mockLogger(), + logger: loggingSystemMock.create().get(), pollInterval$: of(pollInterval), bufferCapacity, work, @@ -288,7 +289,7 @@ describe('TaskPoller', () => { const handler = jest.fn(); const pollRequests$ = new Subject>(); createTaskPoller({ - logger: mockLogger(), + logger: loggingSystemMock.create().get(), pollInterval$: of(pollInterval), bufferCapacity, work: async (...args) => { @@ -339,7 +340,7 @@ describe('TaskPoller', () => { type ResolvableTupple = [string, PromiseLike & Resolvable]; const pollRequests$ = new Subject>(); createTaskPoller<[string, Resolvable], string[]>({ - logger: mockLogger(), + logger: loggingSystemMock.create().get(), pollInterval$: of(pollInterval), bufferCapacity, work: async (...resolvables) => { @@ -399,7 +400,7 @@ describe('TaskPoller', () => { const handler = jest.fn(); const pollRequests$ = new Subject>(); createTaskPoller({ - logger: mockLogger(), + logger: loggingSystemMock.create().get(), pollInterval$: of(pollInterval), bufferCapacity, work: async (...args) => { @@ -440,7 +441,7 @@ describe('TaskPoller', () => { return callCount; }); createTaskPoller({ - logger: mockLogger(), + logger: loggingSystemMock.create().get(), pollInterval$: of(pollInterval), bufferCapacity, work, @@ -483,7 +484,7 @@ describe('TaskPoller', () => { const work = jest.fn(async () => {}); const pollRequests$ = new Subject>(); createTaskPoller({ - logger: mockLogger(), + logger: loggingSystemMock.create().get(), pollInterval$: of(pollInterval), bufferCapacity, work, diff --git a/x-pack/plugins/task_manager/server/polling_lifecycle.test.ts b/x-pack/plugins/task_manager/server/polling_lifecycle.test.ts index 29c8e836303f8c..5f2e774177fd47 100644 --- a/x-pack/plugins/task_manager/server/polling_lifecycle.test.ts +++ b/x-pack/plugins/task_manager/server/polling_lifecycle.test.ts @@ -28,6 +28,16 @@ describe('TaskPollingLifecycle', () => { poll_interval: 6000000, max_poll_inactivity_cycles: 10, request_capacity: 1000, + monitored_aggregated_stats_refresh_rate: 5000, + monitored_stats_required_freshness: 5000, + monitored_stats_running_average_window: 50, + monitored_task_execution_thresholds: { + default: { + error_threshold: 90, + warn_threshold: 80, + }, + custom: {}, + }, }, taskStore: mockTaskStore, logger: taskManagerLogger, diff --git a/x-pack/plugins/task_manager/server/polling_lifecycle.ts b/x-pack/plugins/task_manager/server/polling_lifecycle.ts index 8a506cca699dec..ba19cb63fffa28 100644 --- a/x-pack/plugins/task_manager/server/polling_lifecycle.ts +++ b/x-pack/plugins/task_manager/server/polling_lifecycle.ts @@ -9,6 +9,7 @@ import { performance } from 'perf_hooks'; import { pipe } from 'fp-ts/lib/pipeable'; import { Option, some, map as mapOptional } from 'fp-ts/lib/Option'; +import { tap } from 'rxjs/operators'; import { Logger } from '../../../../src/core/server'; import { Result, asErr, mapErr } from './lib/result_type'; @@ -21,6 +22,8 @@ import { TaskClaim, TaskRunRequest, asTaskRunRequestEvent, + TaskPollingCycle, + asTaskPollingCycleEvent, } from './task_events'; import { fillPool, FillPoolResult } from './lib/fill_pool'; import { Middleware } from './lib/middleware'; @@ -47,7 +50,12 @@ export type TaskPollingLifecycleOpts = { middleware: Middleware; } & ManagedConfiguration; -export type TaskLifecycleEvent = TaskMarkRunning | TaskRun | TaskClaim | TaskRunRequest; +export type TaskLifecycleEvent = + | TaskMarkRunning + | TaskRun + | TaskClaim + | TaskRunRequest + | TaskPollingCycle; /** * The public interface into the task manager system. @@ -181,17 +189,23 @@ export class TaskPollingLifecycle { */ public start() { if (!this.isStarted) { - this.pollingSubscription = this.poller$.subscribe( - mapErr((error: PollingError) => { - if (error.type === PollingErrorType.RequestCapacityReached) { - pipe( - error.data, - mapOptional((id) => this.emitEvent(asTaskRunRequestEvent(id, asErr(error)))) - ); - } - this.logger.error(error.message); - }) - ); + this.pollingSubscription = this.poller$ + .pipe( + tap( + mapErr((error: PollingError) => { + if (error.type === PollingErrorType.RequestCapacityReached) { + pipe( + error.data, + mapOptional((id) => this.emitEvent(asTaskRunRequestEvent(id, asErr(error)))) + ); + } + this.logger.error(error.message); + }) + ) + ) + .subscribe((event: Result>) => { + this.emitEvent(asTaskPollingCycleEvent(event)); + }); } } diff --git a/x-pack/plugins/task_manager/server/queries/mark_available_tasks_as_claimed.test.ts b/x-pack/plugins/task_manager/server/queries/mark_available_tasks_as_claimed.test.ts index 7cdbd8b11bb064..8a94ae4ed82f5c 100644 --- a/x-pack/plugins/task_manager/server/queries/mark_available_tasks_as_claimed.test.ts +++ b/x-pack/plugins/task_manager/server/queries/mark_available_tasks_as_claimed.test.ts @@ -5,21 +5,12 @@ */ import _ from 'lodash'; -import { - asUpdateByQuery, - shouldBeOneOf, - mustBeAllOf, - ExistsFilter, - TermFilter, - RangeFilter, -} from './query_clauses'; +import { asUpdateByQuery, shouldBeOneOf, mustBeAllOf } from './query_clauses'; import { - updateFields, + updateFieldsAndMarkAsFailed, IdleTaskWithExpiredRunAt, RunningOrClaimingTaskWithExpiredRetryAt, - TaskWithSchedule, - taskWithLessThanMaxAttempts, SortByRunAtAndRetryAt, } from './mark_available_tasks_as_claimed'; @@ -40,29 +31,29 @@ describe('mark_available_tasks_as_claimed', () => { createTaskRunner: () => ({ run: () => Promise.resolve() }), }, }); + const claimTasksById = undefined; const defaultMaxAttempts = 1; const taskManagerId = '3478fg6-82374f6-83467gf5-384g6f'; const claimOwnershipUntil = '2019-02-12T21:01:22.479Z'; + const fieldUpdates = { + ownerId: taskManagerId, + retryAt: claimOwnershipUntil, + }; expect( asUpdateByQuery({ query: mustBeAllOf( // Either a task with idle status and runAt <= now or // status running or claiming with a retryAt <= now. - shouldBeOneOf(IdleTaskWithExpiredRunAt, RunningOrClaimingTaskWithExpiredRetryAt), - // Either task has an schedule or the attempts < the maximum configured - shouldBeOneOf( - TaskWithSchedule, - ...Array.from(definitions).map(([type, { maxAttempts }]) => - taskWithLessThanMaxAttempts(type, maxAttempts || defaultMaxAttempts) - ) - ) + shouldBeOneOf(IdleTaskWithExpiredRunAt, RunningOrClaimingTaskWithExpiredRetryAt) + ), + update: updateFieldsAndMarkAsFailed( + fieldUpdates, + claimTasksById || [], + Array.from(definitions).reduce((accumulator, [type, { maxAttempts }]) => { + return { ...accumulator, [type]: maxAttempts || defaultMaxAttempts }; + }, {}) ), - update: updateFields({ - ownerId: taskManagerId, - status: 'claiming', - retryAt: claimOwnershipUntil, - }), sort: SortByRunAtAndRetryAt, }) ).toEqual({ @@ -100,42 +91,6 @@ describe('mark_available_tasks_as_claimed', () => { ], }, }, - // Either task has an recurring schedule or the attempts < the maximum configured - { - bool: { - should: [ - { exists: { field: 'task.schedule' } }, - { - bool: { - must: [ - { term: { 'task.taskType': 'sampleTask' } }, - { - range: { - 'task.attempts': { - lt: 5, - }, - }, - }, - ], - }, - }, - { - bool: { - must: [ - { term: { 'task.taskType': 'otherTask' } }, - { - range: { - 'task.attempts': { - lt: 1, - }, - }, - }, - ], - }, - }, - ], - }, - }, ], }, }, @@ -158,12 +113,26 @@ if (doc['task.runAt'].size()!=0) { }, seq_no_primary_term: true, script: { - source: `ctx._source.task.ownerId=params.ownerId; ctx._source.task.status=params.status; ctx._source.task.retryAt=params.retryAt;`, + source: ` + if (ctx._source.task.schedule != null || ctx._source.task.attempts < params.taskMaxAttempts[ctx._source.task.taskType] || params.claimTasksById.contains(ctx._id)) { + ctx._source.task.status = "claiming"; ${Object.keys(fieldUpdates) + .map((field) => `ctx._source.task.${field}=params.fieldUpdates.${field};`) + .join(' ')} + } else { + ctx._source.task.status = "failed"; + } + `, lang: 'painless', params: { - ownerId: taskManagerId, - retryAt: claimOwnershipUntil, - status: 'claiming', + fieldUpdates: { + ownerId: taskManagerId, + retryAt: claimOwnershipUntil, + }, + claimTasksById: [], + taskMaxAttempts: { + sampleTask: 5, + otherTask: 1, + }, }, }, }); diff --git a/x-pack/plugins/task_manager/server/queries/mark_available_tasks_as_claimed.ts b/x-pack/plugins/task_manager/server/queries/mark_available_tasks_as_claimed.ts index 699af3ed07d5de..072ec4648201a1 100644 --- a/x-pack/plugins/task_manager/server/queries/mark_available_tasks_as_claimed.ts +++ b/x-pack/plugins/task_manager/server/queries/mark_available_tasks_as_claimed.ts @@ -100,12 +100,26 @@ if (doc['task.runAt'].size()!=0) { }, }; -export const updateFields = (fieldUpdates: { - [field: string]: string | number | Date; -}): ScriptClause => ({ - source: Object.keys(fieldUpdates) - .map((field) => `ctx._source.task.${field}=params.${field};`) - .join(' '), +export const updateFieldsAndMarkAsFailed = ( + fieldUpdates: { + [field: string]: string | number | Date; + }, + claimTasksById: string[], + taskMaxAttempts: { [field: string]: number } +): ScriptClause => ({ + source: ` + if (ctx._source.task.schedule != null || ctx._source.task.attempts < params.taskMaxAttempts[ctx._source.task.taskType] || params.claimTasksById.contains(ctx._id)) { + ctx._source.task.status = "claiming"; ${Object.keys(fieldUpdates) + .map((field) => `ctx._source.task.${field}=params.fieldUpdates.${field};`) + .join(' ')} + } else { + ctx._source.task.status = "failed"; + } + `, lang: 'painless', - params: fieldUpdates, + params: { + fieldUpdates, + claimTasksById, + taskMaxAttempts, + }, }); diff --git a/x-pack/plugins/task_manager/server/queries/query_clauses.ts b/x-pack/plugins/task_manager/server/queries/query_clauses.ts index f16ee302d6c83c..5503b9cd941058 100644 --- a/x-pack/plugins/task_manager/server/queries/query_clauses.ts +++ b/x-pack/plugins/task_manager/server/queries/query_clauses.ts @@ -165,7 +165,12 @@ export interface ScriptClause { source: string; lang: string; params: { - [field: string]: string | number | Date; + [field: string]: + | string + | number + | Date + | string[] + | { [field: string]: string | number | Date }; }; } diff --git a/x-pack/plugins/task_manager/server/routes/_mock_handler_arguments.ts b/x-pack/plugins/task_manager/server/routes/_mock_handler_arguments.ts new file mode 100644 index 00000000000000..c9f4de25afaf7b --- /dev/null +++ b/x-pack/plugins/task_manager/server/routes/_mock_handler_arguments.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { RequestHandlerContext, KibanaRequest, KibanaResponseFactory } from 'kibana/server'; +import { identity } from 'lodash'; +import { httpServerMock } from '../../../../../src/core/server/mocks'; + +export function mockHandlerArguments( + {}: {}, + req: unknown, + res?: Array> +): [RequestHandlerContext, KibanaRequest, KibanaResponseFactory] { + return [ + ({} as unknown) as RequestHandlerContext, + req as KibanaRequest, + mockResponseFactory(res), + ]; +} + +export const mockResponseFactory = (resToMock: Array> = []) => { + const factory: jest.Mocked = httpServerMock.createResponseFactory(); + resToMock.forEach((key: string) => { + if (key in factory) { + Object.defineProperty(factory, key, { + value: jest.fn(identity), + }); + } + }); + return (factory as unknown) as KibanaResponseFactory; +}; diff --git a/x-pack/plugins/task_manager/server/routes/health.test.ts b/x-pack/plugins/task_manager/server/routes/health.test.ts new file mode 100644 index 00000000000000..5a0cef8eda94b0 --- /dev/null +++ b/x-pack/plugins/task_manager/server/routes/health.test.ts @@ -0,0 +1,393 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Observable, of, Subject } from 'rxjs'; +import { take } from 'rxjs/operators'; +import { merge } from 'lodash'; +import uuid from 'uuid'; +import { httpServiceMock } from 'src/core/server/mocks'; +import { healthRoute } from './health'; +import { mockHandlerArguments } from './_mock_handler_arguments'; +import { sleep } from '../test_utils'; +import { loggingSystemMock } from '../../../../../src/core/server/mocks'; +import { Logger } from '../../../../../src/core/server'; +import { MonitoringStats, summarizeMonitoringStats } from '../monitoring'; +import { ServiceStatusLevels } from 'src/core/server'; +import { configSchema, TaskManagerConfig } from '../config'; + +describe('healthRoute', () => { + beforeEach(() => { + jest.resetAllMocks(); + }); + + it('registers the route', async () => { + const router = httpServiceMock.createRouter(); + + const logger = loggingSystemMock.create().get(); + healthRoute(router, of(), logger, uuid.v4(), getTaskManagerConfig()); + + const [config] = router.get.mock.calls[0]; + + expect(config.path).toMatchInlineSnapshot(`"/api/task_manager/_health"`); + }); + + it('logs the Task Manager stats at a fixed interval', async () => { + const router = httpServiceMock.createRouter(); + const logger = loggingSystemMock.create().get(); + + const mockStat = mockHealthStats(); + await sleep(10); + const skippedMockStat = mockHealthStats(); + await sleep(10); + const nextMockStat = mockHealthStats(); + + const stats$ = new Subject(); + + const id = uuid.v4(); + healthRoute( + router, + stats$, + logger, + id, + getTaskManagerConfig({ + monitored_stats_required_freshness: 1000, + monitored_aggregated_stats_refresh_rate: 60000, + }) + ); + + stats$.next(mockStat); + await sleep(500); + stats$.next(skippedMockStat); + await sleep(600); + stats$.next(nextMockStat); + + const firstDebug = JSON.parse( + (logger as jest.Mocked).debug.mock.calls[0][0].replace('Latest Monitored Stats: ', '') + ); + expect(firstDebug).toMatchObject({ + id, + timestamp: expect.any(String), + status: expect.any(String), + ...summarizeMonitoringStats(mockStat, getTaskManagerConfig({})), + }); + + const secondDebug = JSON.parse( + (logger as jest.Mocked).debug.mock.calls[1][0].replace('Latest Monitored Stats: ', '') + ); + expect(secondDebug).not.toMatchObject({ + id, + timestamp: expect.any(String), + status: expect.any(String), + ...summarizeMonitoringStats(skippedMockStat, getTaskManagerConfig({})), + }); + expect(secondDebug).toMatchObject({ + id, + timestamp: expect.any(String), + status: expect.any(String), + ...summarizeMonitoringStats(nextMockStat, getTaskManagerConfig({})), + }); + + expect(logger.debug).toHaveBeenCalledTimes(2); + }); + + it('returns a error status if the overall stats have not been updated within the required hot freshness', async () => { + const router = httpServiceMock.createRouter(); + + const stats$ = new Subject(); + + const serviceStatus$ = healthRoute( + router, + stats$, + loggingSystemMock.create().get(), + uuid.v4(), + getTaskManagerConfig({ + monitored_stats_required_freshness: 1000, + monitored_aggregated_stats_refresh_rate: 60000, + }) + ); + + const serviceStatus = getLatest(serviceStatus$); + + const [, handler] = router.get.mock.calls[0]; + + const [context, req, res] = mockHandlerArguments({}, {}, ['ok', 'internalError']); + + await sleep(0); + + stats$.next( + mockHealthStats({ + last_update: new Date(Date.now() - 1500).toISOString(), + }) + ); + + expect(await handler(context, req, res)).toMatchObject({ + body: { + status: 'error', + ...summarizeMonitoringStats( + mockHealthStats({ + last_update: expect.any(String), + stats: { + configuration: { + timestamp: expect.any(String), + }, + workload: { + timestamp: expect.any(String), + }, + runtime: { + timestamp: expect.any(String), + value: { + polling: { + last_successful_poll: expect.any(String), + }, + }, + }, + }, + }), + getTaskManagerConfig({}) + ), + }, + }); + + expect(await serviceStatus).toMatchObject({ + level: ServiceStatusLevels.unavailable, + summary: 'Task Manager is unavailable', + meta: { + status: 'error', + ...summarizeMonitoringStats( + mockHealthStats({ + last_update: expect.any(String), + stats: { + configuration: { + timestamp: expect.any(String), + }, + workload: { + timestamp: expect.any(String), + }, + runtime: { + timestamp: expect.any(String), + value: { + polling: { + last_successful_poll: expect.any(String), + }, + }, + }, + }, + }), + getTaskManagerConfig({}) + ), + }, + }); + }); + + it('returns a error status if the workload stats have not been updated within the required cold freshness', async () => { + const router = httpServiceMock.createRouter(); + + const stats$ = new Subject(); + + healthRoute( + router, + stats$, + loggingSystemMock.create().get(), + uuid.v4(), + getTaskManagerConfig({ + monitored_stats_required_freshness: 5000, + monitored_aggregated_stats_refresh_rate: 60000, + }) + ); + + await sleep(0); + + const lastUpdateOfWorkload = new Date(Date.now() - 120000).toISOString(); + stats$.next( + mockHealthStats({ + stats: { + workload: { + timestamp: lastUpdateOfWorkload, + }, + }, + }) + ); + + const [, handler] = router.get.mock.calls[0]; + + const [context, req, res] = mockHandlerArguments({}, {}, ['ok', 'internalError']); + + await sleep(2000); + + expect(await handler(context, req, res)).toMatchObject({ + body: { + status: 'error', + ...summarizeMonitoringStats( + mockHealthStats({ + last_update: expect.any(String), + stats: { + configuration: { + timestamp: expect.any(String), + }, + workload: { + timestamp: expect.any(String), + }, + runtime: { + timestamp: expect.any(String), + value: { + polling: { + last_successful_poll: expect.any(String), + }, + }, + }, + }, + }), + getTaskManagerConfig() + ), + }, + }); + }); + + it('returns a error status if the poller hasnt polled within the required hot freshness', async () => { + const router = httpServiceMock.createRouter(); + + const stats$ = new Subject(); + healthRoute( + router, + stats$, + loggingSystemMock.create().get(), + uuid.v4(), + getTaskManagerConfig({ + monitored_stats_required_freshness: 1000, + monitored_aggregated_stats_refresh_rate: 60000, + }) + ); + + await sleep(0); + + // eslint-disable-next-line @typescript-eslint/naming-convention + const last_successful_poll = new Date(Date.now() - 2000).toISOString(); + stats$.next( + mockHealthStats({ + stats: { + runtime: { + value: { + polling: { + last_successful_poll, + }, + }, + }, + }, + }) + ); + + const [, handler] = router.get.mock.calls[0]; + + const [context, req, res] = mockHandlerArguments({}, {}, ['ok', 'internalError']); + + expect(await handler(context, req, res)).toMatchObject({ + body: { + status: 'error', + ...summarizeMonitoringStats( + mockHealthStats({ + last_update: expect.any(String), + stats: { + configuration: { + timestamp: expect.any(String), + }, + workload: { + timestamp: expect.any(String), + }, + runtime: { + timestamp: expect.any(String), + value: { + polling: { + last_successful_poll, + }, + }, + }, + }, + }), + getTaskManagerConfig() + ), + }, + }); + }); +}); + +function mockHealthStats(overrides = {}) { + return (merge( + { + last_update: new Date().toISOString(), + stats: { + configuration: { + timestamp: new Date().toISOString(), + value: { + value: { + max_workers: 10, + poll_interval: 6000000, + max_poll_inactivity_cycles: 10, + request_capacity: 1000, + monitored_aggregated_stats_refresh_rate: 5000, + monitored_stats_running_average_window: 50, + monitored_task_execution_thresholds: { + default: { + error_threshold: 90, + warn_threshold: 80, + }, + custom: {}, + }, + }, + }, + }, + workload: { + timestamp: new Date().toISOString(), + value: { + count: 4, + taskTypes: { + actions_telemetry: { count: 2, status: { idle: 2 } }, + alerting_telemetry: { count: 1, status: { idle: 1 } }, + session_cleanup: { count: 1, status: { idle: 1 } }, + }, + schedule: {}, + overdue: 0, + estimatedScheduleDensity: [], + }, + }, + runtime: { + timestamp: new Date().toISOString(), + value: { + drift: [1000, 60000], + execution: { + duration: [], + result_frequency_percent_as_number: [], + }, + polling: { + last_successful_poll: new Date().toISOString(), + result_frequency_percent_as_number: [ + 'NoTasksClaimed', + 'NoTasksClaimed', + 'NoTasksClaimed', + ], + }, + }, + }, + }, + }, + overrides + ) as unknown) as MonitoringStats; +} + +async function getLatest(stream$: Observable) { + return new Promise((resolve) => stream$.pipe(take(1)).subscribe((stats) => resolve(stats))); +} + +const getTaskManagerConfig = (overrides: Partial = {}) => + configSchema.validate( + overrides.monitored_stats_required_freshness + ? { + // use `monitored_stats_required_freshness` as poll interval otherwise we might + // fail validation as it must be greather than the poll interval + poll_interval: overrides.monitored_stats_required_freshness, + ...overrides, + } + : overrides + ); diff --git a/x-pack/plugins/task_manager/server/routes/health.ts b/x-pack/plugins/task_manager/server/routes/health.ts new file mode 100644 index 00000000000000..8ddd728063d2ac --- /dev/null +++ b/x-pack/plugins/task_manager/server/routes/health.ts @@ -0,0 +1,169 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { + IRouter, + RequestHandlerContext, + KibanaRequest, + IKibanaResponse, + KibanaResponseFactory, +} from 'kibana/server'; +import { Observable, Subject } from 'rxjs'; +import { tap, map } from 'rxjs/operators'; +import { throttleTime } from 'rxjs/operators'; +import { isString } from 'lodash'; +import { JsonValue } from 'src/plugins/kibana_utils/common'; +import { Logger, ServiceStatus, ServiceStatusLevels } from '../../../../../src/core/server'; +import { + MonitoringStats, + summarizeMonitoringStats, + HealthStatus, + RawMonitoringStats, +} from '../monitoring'; +import { TaskManagerConfig } from '../config'; + +type MonitoredHealth = RawMonitoringStats & { id: string; status: HealthStatus; timestamp: string }; + +const LEVEL_SUMMARY = { + [ServiceStatusLevels.available.toString()]: 'Task Manager is healthy', + [ServiceStatusLevels.degraded.toString()]: 'Task Manager is unhealthy', + [ServiceStatusLevels.unavailable.toString()]: 'Task Manager is unavailable', +}; + +export function healthRoute( + router: IRouter, + monitoringStats$: Observable, + logger: Logger, + taskManagerId: string, + config: TaskManagerConfig +): Observable { + // if "hot" health stats are any more stale than monitored_stats_required_freshness (pollInterval +1s buffer by default) + // consider the system unhealthy + const requiredHotStatsFreshness: number = config.monitored_stats_required_freshness; + + // if "cold" health stats are any more stale than the configured refresh (+ a buffer), consider the system unhealthy + const requiredColdStatsFreshness: number = config.monitored_aggregated_stats_refresh_rate * 1.5; + + function calculateStatus(monitoredStats: MonitoringStats): MonitoredHealth { + const now = Date.now(); + const timestamp = new Date(now).toISOString(); + const summarizedStats = summarizeMonitoringStats(monitoredStats, config); + + /** + * If the monitored stats aren't fresh, return a red status + */ + const healthStatus = + hasStatus(summarizedStats.stats, HealthStatus.Error) || + hasExpiredHotTimestamps(summarizedStats, now, requiredHotStatsFreshness) || + hasExpiredColdTimestamps(summarizedStats, now, requiredColdStatsFreshness) + ? HealthStatus.Error + : hasStatus(summarizedStats.stats, HealthStatus.Warning) + ? HealthStatus.Warning + : HealthStatus.OK; + return { id: taskManagerId, timestamp, status: healthStatus, ...summarizedStats }; + } + + const serviceStatus$: Subject = new Subject(); + + /* keep track of last health summary, as we'll return that to the next call to _health */ + let lastMonitoredStats: MonitoringStats | null = null; + + /* Log Task Manager stats as a Debug log line at a fixed interval */ + monitoringStats$ + .pipe( + throttleTime(requiredHotStatsFreshness), + tap((stats) => { + lastMonitoredStats = stats; + }), + // Only calculate the summerized stats (calculates all runnign averages and evaluates state) + // when needed by throttling down to the requiredHotStatsFreshness + map((stats) => withServiceStatus(calculateStatus(stats))) + ) + .subscribe(([monitoredHealth, serviceStatus]) => { + serviceStatus$.next(serviceStatus); + logger.debug(`Latest Monitored Stats: ${JSON.stringify(monitoredHealth)}`); + }); + + router.get( + { + path: '/api/task_manager/_health', + validate: false, + }, + async function ( + context: RequestHandlerContext, + req: KibanaRequest, + res: KibanaResponseFactory + ): Promise { + return res.ok({ + body: lastMonitoredStats + ? calculateStatus(lastMonitoredStats) + : { id: taskManagerId, timestamp: new Date().toISOString(), status: HealthStatus.Error }, + }); + } + ); + return serviceStatus$; +} + +export function withServiceStatus( + monitoredHealth: MonitoredHealth +): [MonitoredHealth, ServiceStatus] { + const level = + monitoredHealth.status === HealthStatus.OK + ? ServiceStatusLevels.available + : monitoredHealth.status === HealthStatus.Warning + ? ServiceStatusLevels.degraded + : ServiceStatusLevels.unavailable; + return [ + monitoredHealth, + { + level, + summary: LEVEL_SUMMARY[level.toString()], + meta: monitoredHealth, + }, + ]; +} + +/** + * If certain "hot" stats are not fresh, then the _health api will should return a Red status + * @param monitoringStats The monitored stats + * @param now The time to compare against + * @param requiredFreshness How fresh should these stats be + */ +function hasExpiredHotTimestamps( + monitoringStats: RawMonitoringStats, + now: number, + requiredFreshness: number +): boolean { + return ( + now - + getOldestTimestamp( + monitoringStats.last_update, + monitoringStats.stats.runtime?.value.polling.last_successful_poll + ) > + requiredFreshness + ); +} + +function hasExpiredColdTimestamps( + monitoringStats: RawMonitoringStats, + now: number, + requiredFreshness: number +): boolean { + return now - getOldestTimestamp(monitoringStats.stats.workload?.timestamp) > requiredFreshness; +} + +function hasStatus(stats: RawMonitoringStats['stats'], status: HealthStatus): boolean { + return Object.values(stats) + .map((stat) => stat?.status === status) + .includes(true); +} + +function getOldestTimestamp(...timestamps: Array): number { + const validTimestamps = timestamps + .map((timestamp) => (isString(timestamp) ? Date.parse(timestamp) : NaN)) + .filter((timestamp) => !isNaN(timestamp)); + return validTimestamps.length ? Math.min(...validTimestamps) : 0; +} diff --git a/x-pack/plugins/task_manager/server/routes/index.ts b/x-pack/plugins/task_manager/server/routes/index.ts new file mode 100644 index 00000000000000..4fa1aa6cb7a9be --- /dev/null +++ b/x-pack/plugins/task_manager/server/routes/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export { healthRoute } from './health'; diff --git a/x-pack/plugins/task_manager/server/task_events.ts b/x-pack/plugins/task_manager/server/task_events.ts index e1dd85f868cdd2..b011d435e28dc0 100644 --- a/x-pack/plugins/task_manager/server/task_events.ts +++ b/x-pack/plugins/task_manager/server/task_events.ts @@ -9,63 +9,109 @@ import { Option } from 'fp-ts/lib/Option'; import { ConcreteTaskInstance } from './task'; import { Result, Err } from './lib/result_type'; +import { FillPoolResult } from './lib/fill_pool'; +import { PollingError } from './polling'; +import { TaskRunResult } from './task_runner'; export enum TaskEventType { TASK_CLAIM = 'TASK_CLAIM', TASK_MARK_RUNNING = 'TASK_MARK_RUNNING', TASK_RUN = 'TASK_RUN', TASK_RUN_REQUEST = 'TASK_RUN_REQUEST', + TASK_POLLING_CYCLE = 'TASK_POLLING_CYCLE', +} + +export interface TaskTiming { + start: number; + stop: number; +} + +export function startTaskTimer(): () => TaskTiming { + const start = Date.now(); + return () => ({ start, stop: Date.now() }); } export interface TaskEvent { - id: string; + id?: string; + timing?: TaskTiming; type: TaskEventType; event: Result; } +export interface RanTask { + task: ConcreteTaskInstance; + result: TaskRunResult; +} +export type ErroredTask = RanTask & { + error: Error; +}; + export type TaskMarkRunning = TaskEvent; -export type TaskRun = TaskEvent; +export type TaskRun = TaskEvent; export type TaskClaim = TaskEvent>; export type TaskRunRequest = TaskEvent; +export type TaskPollingCycle = TaskEvent>; export function asTaskMarkRunningEvent( id: string, - event: Result + event: Result, + timing?: TaskTiming ): TaskMarkRunning { return { id, type: TaskEventType.TASK_MARK_RUNNING, event, + timing, }; } -export function asTaskRunEvent(id: string, event: Result): TaskRun { +export function asTaskRunEvent( + id: string, + event: Result, + timing?: TaskTiming +): TaskRun { return { id, type: TaskEventType.TASK_RUN, event, + timing, }; } export function asTaskClaimEvent( id: string, - event: Result> + event: Result>, + timing?: TaskTiming ): TaskClaim { return { id, type: TaskEventType.TASK_CLAIM, event, + timing, }; } export function asTaskRunRequestEvent( id: string, // we only emit a TaskRunRequest event when it fails - event: Err + event: Err, + timing?: TaskTiming ): TaskRunRequest { return { id, type: TaskEventType.TASK_RUN_REQUEST, event, + timing, + }; +} + +export function asTaskPollingCycleEvent( + event: Result>, + timing?: TaskTiming +): TaskPollingCycle { + return { + type: TaskEventType.TASK_POLLING_CYCLE, + event, + timing, }; } @@ -85,3 +131,8 @@ export function isTaskRunRequestEvent( ): taskEvent is TaskRunRequest { return taskEvent.type === TaskEventType.TASK_RUN_REQUEST; } +export function isTaskPollingCycleEvent( + taskEvent: TaskEvent +): taskEvent is TaskPollingCycle { + return taskEvent.type === TaskEventType.TASK_POLLING_CYCLE; +} diff --git a/x-pack/plugins/task_manager/server/task_pool.test.ts b/x-pack/plugins/task_manager/server/task_pool.test.ts index 12b731b2b78ae6..a174af71ef18f2 100644 --- a/x-pack/plugins/task_manager/server/task_pool.test.ts +++ b/x-pack/plugins/task_manager/server/task_pool.test.ts @@ -7,7 +7,9 @@ import sinon from 'sinon'; import { of, Subject } from 'rxjs'; import { TaskPool, TaskPoolRunResult } from './task_pool'; -import { mockLogger, resolvable, sleep } from './test_utils'; +import { resolvable, sleep } from './test_utils'; +import { loggingSystemMock } from '../../../../src/core/server/mocks'; +import { Logger } from '../../../../src/core/server'; import { asOk } from './lib/result_type'; import { SavedObjectsErrorHelpers } from '../../../../src/core/server'; import moment from 'moment'; @@ -16,7 +18,7 @@ describe('TaskPool', () => { test('occupiedWorkers are a sum of running tasks', async () => { const pool = new TaskPool({ maxWorkers$: of(200), - logger: mockLogger(), + logger: loggingSystemMock.create().get(), }); const result = await pool.run([{ ...mockTask() }, { ...mockTask() }, { ...mockTask() }]); @@ -28,7 +30,7 @@ describe('TaskPool', () => { test('availableWorkers are a function of total_capacity - occupiedWorkers', async () => { const pool = new TaskPool({ maxWorkers$: of(10), - logger: mockLogger(), + logger: loggingSystemMock.create().get(), }); const result = await pool.run([{ ...mockTask() }, { ...mockTask() }, { ...mockTask() }]); @@ -41,7 +43,7 @@ describe('TaskPool', () => { const maxWorkers$ = new Subject(); const pool = new TaskPool({ maxWorkers$, - logger: mockLogger(), + logger: loggingSystemMock.create().get(), }); expect(pool.availableWorkers).toEqual(0); @@ -52,7 +54,7 @@ describe('TaskPool', () => { test('does not run tasks that are beyond its available capacity', async () => { const pool = new TaskPool({ maxWorkers$: of(2), - logger: mockLogger(), + logger: loggingSystemMock.create().get(), }); const shouldRun = mockRun(); @@ -71,7 +73,7 @@ describe('TaskPool', () => { }); test('should log when marking a Task as running fails', async () => { - const logger = mockLogger(); + const logger = loggingSystemMock.create().get(); const pool = new TaskPool({ maxWorkers$: of(2), logger, @@ -84,7 +86,7 @@ describe('TaskPool', () => { const result = await pool.run([mockTask(), taskFailedToMarkAsRunning, mockTask()]); - expect(logger.error.mock.calls[0]).toMatchInlineSnapshot(` + expect((logger as jest.Mocked).error.mock.calls[0]).toMatchInlineSnapshot(` Array [ "Failed to mark Task TaskType \\"shooooo\\" as running: Mark Task as running has failed miserably", ] @@ -94,7 +96,7 @@ describe('TaskPool', () => { }); test('should log when running a Task fails', async () => { - const logger = mockLogger(); + const logger = loggingSystemMock.create().get(); const pool = new TaskPool({ maxWorkers$: of(3), logger, @@ -107,7 +109,7 @@ describe('TaskPool', () => { const result = await pool.run([mockTask(), taskFailedToRun, mockTask()]); - expect(logger.warn.mock.calls[0]).toMatchInlineSnapshot(` + expect((logger as jest.Mocked).warn.mock.calls[0]).toMatchInlineSnapshot(` Array [ "Task TaskType \\"shooooo\\" failed in attempt to run: Run Task has failed miserably", ] @@ -117,7 +119,7 @@ describe('TaskPool', () => { }); test('should not log when running a Task fails due to the Task SO having been deleted while in flight', async () => { - const logger = mockLogger(); + const logger = loggingSystemMock.create().get(); const pool = new TaskPool({ maxWorkers$: of(3), logger, @@ -139,7 +141,7 @@ describe('TaskPool', () => { }); test('Running a task which fails still takes up capacity', async () => { - const logger = mockLogger(); + const logger = loggingSystemMock.create().get(); const pool = new TaskPool({ maxWorkers$: of(1), logger, @@ -159,7 +161,7 @@ describe('TaskPool', () => { test('clears up capacity when a task completes', async () => { const pool = new TaskPool({ maxWorkers$: of(1), - logger: mockLogger(), + logger: loggingSystemMock.create().get(), }); const firstWork = resolvable(); @@ -202,7 +204,7 @@ describe('TaskPool', () => { }); test('run cancels expired tasks prior to running new tasks', async () => { - const logger = mockLogger(); + const logger = loggingSystemMock.create().get(); const pool = new TaskPool({ maxWorkers$: of(2), logger, @@ -259,7 +261,7 @@ describe('TaskPool', () => { }); test('logs if cancellation errors', async () => { - const logger = mockLogger(); + const logger = loggingSystemMock.create().get(); const pool = new TaskPool({ logger, maxWorkers$: of(20), @@ -290,7 +292,7 @@ describe('TaskPool', () => { // Allow the task to cancel... await cancelled; - expect(logger.error.mock.calls[0][0]).toMatchInlineSnapshot( + expect((logger as jest.Mocked).error.mock.calls[0][0]).toMatchInlineSnapshot( `"Failed to cancel task \\"shooooo!\\": Error: Dern!"` ); }); diff --git a/x-pack/plugins/task_manager/server/task_runner.test.ts b/x-pack/plugins/task_manager/server/task_runner.test.ts index 8fb1df444c603a..676eeedf08f5af 100644 --- a/x-pack/plugins/task_manager/server/task_runner.test.ts +++ b/x-pack/plugins/task_manager/server/task_runner.test.ts @@ -6,16 +6,18 @@ import _ from 'lodash'; import sinon from 'sinon'; -import { minutesFromNow } from './lib/intervals'; +import { secondsFromNow } from './lib/intervals'; import { asOk, asErr } from './lib/result_type'; -import { TaskEvent, asTaskRunEvent, asTaskMarkRunningEvent } from './task_events'; +import { TaskManagerRunner, TaskRunResult } from './task_runner'; +import { TaskEvent, asTaskRunEvent, asTaskMarkRunningEvent, TaskRun } from './task_events'; import { ConcreteTaskInstance, TaskStatus, TaskDefinition, RunResult } from './task'; -import { TaskManagerRunner } from './task_runner'; import { SavedObjectsErrorHelpers } from '../../../../src/core/server'; import moment from 'moment'; import { TaskTypeDictionary } from './task_type_dictionary'; import { mockLogger } from './test_utils'; +const minutesFromNow = (mins: number): Date => secondsFromNow(mins * 60); + let fakeTimer: sinon.SinonFakeTimers; beforeAll(() => { @@ -812,7 +814,9 @@ describe('TaskManagerRunner', () => { await runner.run(); - expect(onTaskEvent).toHaveBeenCalledWith(asTaskRunEvent(id, asOk(instance))); + expect(onTaskEvent).toHaveBeenCalledWith( + withAnyTiming(asTaskRunEvent(id, asOk({ task: instance, result: TaskRunResult.Success }))) + ); }); test('emits TaskEvent when a recurring task is run successfully', async () => { @@ -839,14 +843,16 @@ describe('TaskManagerRunner', () => { await runner.run(); - expect(onTaskEvent).toHaveBeenCalledWith(asTaskRunEvent(id, asOk(instance))); + expect(onTaskEvent).toHaveBeenCalledWith( + withAnyTiming(asTaskRunEvent(id, asOk({ task: instance, result: TaskRunResult.Success }))) + ); }); test('emits TaskEvent when a task run throws an error', async () => { const id = _.random(1, 20).toString(); const error = new Error('Dangit!'); const onTaskEvent = jest.fn(); - const { runner } = testOpts({ + const { runner, instance } = testOpts({ onTaskEvent, instance: { id, @@ -864,7 +870,11 @@ describe('TaskManagerRunner', () => { }); await runner.run(); - expect(onTaskEvent).toHaveBeenCalledWith(asTaskRunEvent(id, asErr(error))); + expect(onTaskEvent).toHaveBeenCalledWith( + withAnyTiming( + asTaskRunEvent(id, asErr({ error, task: instance, result: TaskRunResult.RetryScheduled })) + ) + ); expect(onTaskEvent).toHaveBeenCalledTimes(1); }); @@ -872,7 +882,7 @@ describe('TaskManagerRunner', () => { const id = _.random(1, 20).toString(); const error = new Error('Dangit!'); const onTaskEvent = jest.fn(); - const { runner } = testOpts({ + const { runner, instance } = testOpts({ onTaskEvent, instance: { id, @@ -893,7 +903,11 @@ describe('TaskManagerRunner', () => { await runner.run(); - expect(onTaskEvent).toHaveBeenCalledWith(asTaskRunEvent(id, asErr(error))); + expect(onTaskEvent).toHaveBeenCalledWith( + withAnyTiming( + asTaskRunEvent(id, asErr({ error, task: instance, result: TaskRunResult.RetryScheduled })) + ) + ); expect(onTaskEvent).toHaveBeenCalledTimes(1); }); @@ -901,7 +915,7 @@ describe('TaskManagerRunner', () => { const id = _.random(1, 20).toString(); const error = new Error('Dangit!'); const onTaskEvent = jest.fn(); - const { runner, store } = testOpts({ + const { runner, store, instance: originalInstance } = testOpts({ onTaskEvent, instance: { id, @@ -925,7 +939,18 @@ describe('TaskManagerRunner', () => { const instance = store.update.args[0][0]; expect(instance.status).toBe('failed'); - expect(onTaskEvent).toHaveBeenCalledWith(asTaskRunEvent(id, asErr(error))); + expect(onTaskEvent).toHaveBeenCalledWith( + withAnyTiming( + asTaskRunEvent( + id, + asErr({ + error, + task: originalInstance, + result: TaskRunResult.Failed, + }) + ) + ) + ); expect(onTaskEvent).toHaveBeenCalledTimes(1); }); }); @@ -936,6 +961,13 @@ describe('TaskManagerRunner', () => { onTaskEvent?: (event: TaskEvent) => void; } + function withAnyTiming(taskRun: TaskRun) { + return { + ...taskRun, + timing: { start: expect.any(Number), stop: expect.any(Number) }, + }; + } + function testOpts(opts: TestOpts) { const callCluster = sinon.stub(); const createTaskRunner = sinon.stub(); diff --git a/x-pack/plugins/task_manager/server/task_runner.ts b/x-pack/plugins/task_manager/server/task_runner.ts index 24a487e3660293..45e4cb30005701 100644 --- a/x-pack/plugins/task_manager/server/task_runner.ts +++ b/x-pack/plugins/task_manager/server/task_runner.ts @@ -10,16 +10,23 @@ * rescheduling, middleware application, etc. */ +import { Logger } from 'src/core/server'; import apm from 'elastic-apm-node'; import { performance } from 'perf_hooks'; import Joi from 'joi'; import { identity, defaults, flow } from 'lodash'; -import { Logger } from '../../../../src/core/server'; -import { asOk, asErr, mapErr, eitherAsync, unwrap, mapOk, Result } from './lib/result_type'; -import { TaskRun, TaskMarkRunning, asTaskRunEvent, asTaskMarkRunningEvent } from './task_events'; -import { intervalFromDate, intervalFromNow } from './lib/intervals'; import { Middleware } from './lib/middleware'; +import { asOk, asErr, mapErr, eitherAsync, unwrap, isOk, mapOk, Result } from './lib/result_type'; +import { + TaskRun, + TaskMarkRunning, + asTaskRunEvent, + asTaskMarkRunningEvent, + startTaskTimer, + TaskTiming, +} from './task_events'; +import { intervalFromDate, intervalFromNow } from './lib/intervals'; import { CancelFunction, CancellableTask, @@ -63,6 +70,21 @@ type Opts = { onTaskEvent?: (event: TaskRun | TaskMarkRunning) => void; } & Pick; +export enum TaskRunResult { + // Task completed successfully + Success = 'Success', + // Recurring Task completed successfully + SuccessRescheduled = 'Success', + // // Task completed successfully after a retry + // SuccessfulRetry = 'SuccessfulRetry', + // // Recurring Task completed successfully after a retry + // SuccessfulRetryRescheduled = 'SuccessfulRetry', + // Task has failed and a retry has been scheduled + RetryScheduled = 'RetryScheduled', + // Task has failed + Failed = 'Failed', +} + /** * Runs a background task, ensures that errors are properly handled, * allows for cancellation. @@ -172,6 +194,7 @@ export class TaskManagerRunner implements TaskRunner { taskInstance: this.instance, }); + const stopTaskTimer = startTaskTimer(); const apmTrans = apm.startTransaction( `taskManager run ${this.instance.taskType}`, 'taskManager' @@ -181,13 +204,16 @@ export class TaskManagerRunner implements TaskRunner { const result = await this.task.run(); const validatedResult = this.validateResult(result); if (apmTrans) apmTrans.end('success'); - return this.processResult(validatedResult); + return this.processResult(validatedResult, stopTaskTimer()); } catch (err) { this.logger.error(`Task ${this} failed: ${err}`); // in error scenario, we can not get the RunResult // re-use modifiedContext's state, which is correct as of beforeRun if (apmTrans) apmTrans.end('error'); - return this.processResult(asErr({ error: err, state: modifiedContext.taskInstance.state })); + return this.processResult( + asErr({ error: err, state: modifiedContext.taskInstance.state }), + stopTaskTimer() + ); } } @@ -337,8 +363,9 @@ export class TaskManagerRunner implements TaskRunner { private async processResultForRecurringTask( result: Result - ): Promise { - const fieldUpdates = flow( + ): Promise { + const hasTaskRunFailed = isOk(result); + const fieldUpdates: Partial & Pick = flow( // if running the task has failed ,try to correct by scheduling a retry in the near future mapErr(this.rescheduleFailedRun), // if retrying is possible (new runAt) or this is an recurring task - reschedule @@ -357,7 +384,7 @@ export class TaskManagerRunner implements TaskRunner { await this.bufferedTaskStore.update( defaults( { - ...(fieldUpdates as Partial), + ...fieldUpdates, // reset fields that track the lifecycle of the concluded `task run` startedAt: null, retryAt: null, @@ -366,9 +393,15 @@ export class TaskManagerRunner implements TaskRunner { this.instance ) ); + + return fieldUpdates.status === TaskStatus.Failed + ? TaskRunResult.Failed + : hasTaskRunFailed + ? TaskRunResult.SuccessRescheduled + : TaskRunResult.RetryScheduled; } - private async processResultWhenDone(): Promise { + private async processResultWhenDone(): Promise { // not a recurring task: clean up by removing the task instance from store try { await this.bufferedTaskStore.remove(this.instance.id); @@ -379,24 +412,38 @@ export class TaskManagerRunner implements TaskRunner { throw err; } } + return TaskRunResult.Success; } private async processResult( - result: Result + result: Result, + taskTiming: TaskTiming ): Promise> { + const task = this.instance; await eitherAsync( result, async ({ runAt }: SuccessfulRunResult) => { - if (runAt || this.instance.schedule) { - await this.processResultForRecurringTask(result); - } else { - await this.processResultWhenDone(); - } - this.onTaskEvent(asTaskRunEvent(this.id, asOk(this.instance))); + this.onTaskEvent( + asTaskRunEvent( + this.id, + asOk({ + task, + result: await (runAt || task.schedule + ? this.processResultForRecurringTask(result) + : this.processResultWhenDone()), + }), + taskTiming + ) + ); }, async ({ error }: FailedRunResult) => { - await this.processResultForRecurringTask(result); - this.onTaskEvent(asTaskRunEvent(this.id, asErr(error))); + this.onTaskEvent( + asTaskRunEvent( + this.id, + asErr({ task, result: await this.processResultForRecurringTask(result), error }), + taskTiming + ) + ); } ); return result; diff --git a/x-pack/plugins/task_manager/server/task_scheduling.test.ts b/x-pack/plugins/task_manager/server/task_scheduling.test.ts index 1f7f9250d90147..8d660f57ab8755 100644 --- a/x-pack/plugins/task_manager/server/task_scheduling.test.ts +++ b/x-pack/plugins/task_manager/server/task_scheduling.test.ts @@ -21,6 +21,7 @@ import { asErr, asOk } from './lib/result_type'; import { ConcreteTaskInstance, TaskLifecycleResult, TaskStatus } from './task'; import { createInitialMiddleware } from './lib/middleware'; import { taskStoreMock } from './task_store.mock'; +import { TaskRunResult } from './task_runner'; import { mockLogger } from './test_utils'; describe('TaskScheduling', () => { @@ -113,7 +114,7 @@ describe('TaskScheduling', () => { const result = taskScheduling.runNow(id); const task = { id } as ConcreteTaskInstance; - events$.next(asTaskRunEvent(id, asOk(task))); + events$.next(asTaskRunEvent(id, asOk({ task, result: TaskRunResult.Success }))); return expect(result).resolves.toEqual({ id }); }); @@ -132,7 +133,16 @@ describe('TaskScheduling', () => { const task = { id } as ConcreteTaskInstance; events$.next(asTaskClaimEvent(id, asOk(task))); events$.next(asTaskMarkRunningEvent(id, asOk(task))); - events$.next(asTaskRunEvent(id, asErr(new Error('some thing gone wrong')))); + events$.next( + asTaskRunEvent( + id, + asErr({ + task, + error: new Error('some thing gone wrong'), + result: TaskRunResult.Failed, + }) + ) + ); return expect(result).rejects.toMatchInlineSnapshot( `[Error: Failed to run task "01ddff11-e88a-4d13-bc4e-256164e755e2": Error: some thing gone wrong]` @@ -306,10 +316,20 @@ describe('TaskScheduling', () => { const otherTask = { id: differentTask } as ConcreteTaskInstance; events$.next(asTaskClaimEvent(id, asOk(task))); events$.next(asTaskClaimEvent(differentTask, asOk(otherTask))); + events$.next( + asTaskRunEvent(differentTask, asOk({ task: otherTask, result: TaskRunResult.Success })) + ); - events$.next(asTaskRunEvent(differentTask, asOk(task))); - - events$.next(asTaskRunEvent(id, asErr(new Error('some thing gone wrong')))); + events$.next( + asTaskRunEvent( + id, + asErr({ + task, + error: new Error('some thing gone wrong'), + result: TaskRunResult.Failed, + }) + ) + ); return expect(result).rejects.toMatchInlineSnapshot( `[Error: Failed to run task "01ddff11-e88a-4d13-bc4e-256164e755e2": Error: some thing gone wrong]` diff --git a/x-pack/plugins/task_manager/server/task_scheduling.ts b/x-pack/plugins/task_manager/server/task_scheduling.ts index 00f7d853d71142..9806ada386e4ab 100644 --- a/x-pack/plugins/task_manager/server/task_scheduling.ts +++ b/x-pack/plugins/task_manager/server/task_scheduling.ts @@ -10,7 +10,13 @@ import { Option, map as mapOptional, getOrElse } from 'fp-ts/lib/Option'; import { Logger } from '../../../../src/core/server'; import { asOk, either, map, mapErr, promiseResult } from './lib/result_type'; -import { isTaskRunEvent, isTaskClaimEvent, isTaskRunRequestEvent } from './task_events'; +import { + isTaskRunEvent, + isTaskClaimEvent, + isTaskRunRequestEvent, + RanTask, + ErroredTask, +} from './task_events'; import { Middleware } from './lib/middleware'; import { ConcreteTaskInstance, @@ -23,6 +29,7 @@ import { import { TaskStore } from './task_store'; import { ensureDeprecatedFieldsAreCorrected } from './lib/correct_deprecated_fields'; import { TaskLifecycleEvent, TaskPollingLifecycle } from './polling_lifecycle'; +import { FillPoolResult } from './lib/fill_pool'; const VERSION_CONFLICT_STATUS = 409; @@ -118,16 +125,19 @@ export class TaskScheduling { return reject(await this.identifyTaskFailureReason(taskId, error)); }, taskEvent.event); } else { - either>( + either< + RanTask | ConcreteTaskInstance | FillPoolResult, + Error | ErroredTask | Option + >( taskEvent.event, - (taskInstance: ConcreteTaskInstance) => { + (taskInstance: RanTask | ConcreteTaskInstance | FillPoolResult) => { // resolve if the task has run sucessfully if (isTaskRunEvent(taskEvent)) { subscription.unsubscribe(); - resolve({ id: taskInstance.id }); + resolve({ id: (taskInstance as RanTask).task.id }); } }, - async (error: Error | Option) => { + async (errorResult: Error | ErroredTask | Option) => { // reject if any error event takes place for the requested task subscription.unsubscribe(); return reject( @@ -135,7 +145,9 @@ export class TaskScheduling { `Failed to run task "${taskId}": ${ isTaskRunRequestEvent(taskEvent) ? `Task Manager is at capacity, please try again later` - : error + : isTaskRunEvent(taskEvent) + ? `${(errorResult as ErroredTask).error}` + : `${errorResult}` }` ) ); diff --git a/x-pack/plugins/task_manager/server/task_store.mock.ts b/x-pack/plugins/task_manager/server/task_store.mock.ts index 9b82a3e3ee7ab3..7af1b9ef5f110a 100644 --- a/x-pack/plugins/task_manager/server/task_store.mock.ts +++ b/x-pack/plugins/task_manager/server/task_store.mock.ts @@ -31,6 +31,7 @@ export const taskStoreMock = { get: jest.fn(), getLifecycle: jest.fn(), fetch: jest.fn(), + aggregate: jest.fn(), maxAttempts, index, taskManagerId, diff --git a/x-pack/plugins/task_manager/server/task_store.test.ts b/x-pack/plugins/task_manager/server/task_store.test.ts index a40df3b84132ed..46e55df4ee1e67 100644 --- a/x-pack/plugins/task_manager/server/task_store.test.ts +++ b/x-pack/plugins/task_manager/server/task_store.test.ts @@ -415,41 +415,6 @@ describe('TaskStore', () => { ], }, }, - { - bool: { - should: [ - { exists: { field: 'task.schedule' } }, - { - bool: { - must: [ - { term: { 'task.taskType': 'foo' } }, - { - range: { - 'task.attempts': { - lt: maxAttempts, - }, - }, - }, - ], - }, - }, - { - bool: { - must: [ - { term: { 'task.taskType': 'bar' } }, - { - range: { - 'task.attempts': { - lt: customMaxAttempts, - }, - }, - }, - ], - }, - }, - ], - }, - }, ], }, }, @@ -501,6 +466,11 @@ if (doc['task.runAt'].size()!=0) { const maxAttempts = _.random(2, 43); const customMaxAttempts = _.random(44, 100); const definitions = new TaskTypeDictionary(mockLogger()); + const taskManagerId = uuid.v1(); + const fieldUpdates = { + ownerId: taskManagerId, + retryAt: new Date(Date.now()), + }; definitions.registerTaskDefinitions({ foo: { title: 'foo', @@ -514,10 +484,11 @@ if (doc['task.runAt'].size()!=0) { }); const { args: { - updateByQuery: { body: { query, sort } = {} }, + updateByQuery: { body: { query, script, sort } = {} }, }, } = await testClaimAvailableTasks({ opts: { + taskManagerId, maxAttempts, definitions, }, @@ -576,41 +547,6 @@ if (doc['task.runAt'].size()!=0) { ], }, }, - { - bool: { - should: [ - { exists: { field: 'task.schedule' } }, - { - bool: { - must: [ - { term: { 'task.taskType': 'foo' } }, - { - range: { - 'task.attempts': { - lt: maxAttempts, - }, - }, - }, - ], - }, - }, - { - bool: { - must: [ - { term: { 'task.taskType': 'bar' } }, - { - range: { - 'task.attempts': { - lt: customMaxAttempts, - }, - }, - }, - ], - }, - }, - ], - }, - }, ], }, }, @@ -640,6 +576,30 @@ if (doc['task.runAt'].size()!=0) { }, }); + expect(script).toMatchObject({ + source: ` + if (ctx._source.task.schedule != null || ctx._source.task.attempts < params.taskMaxAttempts[ctx._source.task.taskType] || params.claimTasksById.contains(ctx._id)) { + ctx._source.task.status = "claiming"; ${Object.keys(fieldUpdates) + .map((field) => `ctx._source.task.${field}=params.fieldUpdates.${field};`) + .join(' ')} + } else { + ctx._source.task.status = "failed"; + } + `, + lang: 'painless', + params: { + fieldUpdates, + claimTasksById: [ + 'task:33c6977a-ed6d-43bd-98d9-3f827f7b7cd8', + 'task:a208b22c-14ec-4fb4-995f-d2ff7a3b03b8', + ], + taskMaxAttempts: { + bar: customMaxAttempts, + foo: maxAttempts, + }, + }, + }); + expect(sort).toMatchObject([ '_score', { @@ -665,6 +625,10 @@ if (doc['task.runAt'].size()!=0) { test('it claims tasks by setting their ownerId, status and retryAt', async () => { const taskManagerId = uuid.v1(); const claimOwnershipUntil = new Date(Date.now()); + const fieldUpdates = { + ownerId: taskManagerId, + retryAt: claimOwnershipUntil, + }; const { args: { updateByQuery: { body: { script } = {} }, @@ -679,12 +643,24 @@ if (doc['task.runAt'].size()!=0) { }, }); expect(script).toMatchObject({ - source: `ctx._source.task.ownerId=params.ownerId; ctx._source.task.status=params.status; ctx._source.task.retryAt=params.retryAt;`, + source: ` + if (ctx._source.task.schedule != null || ctx._source.task.attempts < params.taskMaxAttempts[ctx._source.task.taskType] || params.claimTasksById.contains(ctx._id)) { + ctx._source.task.status = "claiming"; ${Object.keys(fieldUpdates) + .map((field) => `ctx._source.task.${field}=params.fieldUpdates.${field};`) + .join(' ')} + } else { + ctx._source.task.status = "failed"; + } + `, lang: 'painless', params: { - ownerId: taskManagerId, - retryAt: claimOwnershipUntil, - status: 'claiming', + fieldUpdates, + claimTasksById: [], + taskMaxAttempts: { + dernstraight: 2, + report: 2, + yawn: 2, + }, }, }); }); diff --git a/x-pack/plugins/task_manager/server/task_store.ts b/x-pack/plugins/task_manager/server/task_store.ts index 63b6ab7412ec53..8c0d7764e009f8 100644 --- a/x-pack/plugins/task_manager/server/task_store.ts +++ b/x-pack/plugins/task_manager/server/task_store.ts @@ -41,26 +41,23 @@ import { shouldBeOneOf, mustBeAllOf, filterDownBy, - ExistsFilter, - TermFilter, - RangeFilter, asPinnedQuery, matchesClauses, SortOptions, } from './queries/query_clauses'; import { - updateFields, + updateFieldsAndMarkAsFailed, IdleTaskWithExpiredRunAt, InactiveTasks, RunningOrClaimingTaskWithExpiredRetryAt, - TaskWithSchedule, - taskWithLessThanMaxAttempts, SortByRunAtAndRetryAt, tasksClaimedByOwner, } from './queries/mark_available_tasks_as_claimed'; import { TaskTypeDictionary } from './task_type_dictionary'; +import { ESSearchResponse, ESSearchBody } from '../../apm/typings/elasticsearch'; + export interface StoreOpts { esClient: ElasticsearchClient; index: string; @@ -79,6 +76,9 @@ export interface SearchOpts { search_after?: unknown[]; } +export type AggregationOpts = Pick, 'aggs'> & + Pick; + export interface UpdateByQuerySearchOpts extends SearchOpts { script?: object; } @@ -259,18 +259,13 @@ export class TaskStore { claimTasksById: OwnershipClaimingOpts['claimTasksById'], size: OwnershipClaimingOpts['size'] ): Promise { - const tasksWithRemainingAttempts = [...this.definitions].map(([type, { maxAttempts }]) => - taskWithLessThanMaxAttempts(type, maxAttempts || this.maxAttempts) - ); + const taskMaxAttempts = [...this.definitions].reduce((accumulator, [type, { maxAttempts }]) => { + return { ...accumulator, [type]: maxAttempts || this.maxAttempts }; + }, {}); const queryForScheduledTasks = mustBeAllOf( // Either a task with idle status and runAt <= now or // status running or claiming with a retryAt <= now. - shouldBeOneOf(IdleTaskWithExpiredRunAt, RunningOrClaimingTaskWithExpiredRetryAt), - // Either task has a schedule or the attempts < the maximum configured - shouldBeOneOf( - TaskWithSchedule, - ...tasksWithRemainingAttempts - ) + shouldBeOneOf(IdleTaskWithExpiredRunAt, RunningOrClaimingTaskWithExpiredRetryAt) ); // The documents should be sorted by runAt/retryAt, unless there are pinned @@ -295,11 +290,14 @@ export class TaskStore { ), filterDownBy(InactiveTasks) ), - update: updateFields({ - ownerId: this.taskManagerId, - status: 'claiming', - retryAt: claimOwnershipUntil, - }), + update: updateFieldsAndMarkAsFailed( + { + ownerId: this.taskManagerId, + retryAt: claimOwnershipUntil, + }, + claimTasksById || [], + taskMaxAttempts + ), sort, }), { @@ -501,6 +499,25 @@ export class TaskStore { } } + public async aggregate({ + aggs, + query, + size = 0, + }: TSearchRequest): Promise> { + const { body } = await this.esClient.search< + ESSearchResponse + >({ + index: this.index, + ignore_unavailable: true, + body: ensureAggregationOnlyReturnsTaskObjects({ + query, + aggs, + size, + }), + }); + return body; + } + private async updateByQuery( opts: UpdateByQuerySearchOpts = {}, // eslint-disable-next-line @typescript-eslint/naming-convention @@ -586,6 +603,22 @@ function ensureQueryOnlyReturnsTaskObjects(opts: SearchOpts): SearchOpts { }; } +function ensureAggregationOnlyReturnsTaskObjects(opts: AggregationOpts): AggregationOpts { + const originalQuery = opts.query; + const filterToOnlyTasks = { + bool: { + filter: [{ term: { type: 'task' } }], + }, + }; + const query = originalQuery + ? { bool: { must: [filterToOnlyTasks, originalQuery] } } + : filterToOnlyTasks; + return { + ...opts, + query, + }; +} + function isSavedObjectsUpdateResponse( result: SavedObjectsUpdateResponse | Error ): result is SavedObjectsUpdateResponse { diff --git a/x-pack/plugins/telemetry_collection_xpack/tsconfig.json b/x-pack/plugins/telemetry_collection_xpack/tsconfig.json new file mode 100644 index 00000000000000..476f5926f757a4 --- /dev/null +++ b/x-pack/plugins/telemetry_collection_xpack/tsconfig.json @@ -0,0 +1,21 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "composite": true, + "outDir": "./target/types", + "emitDeclarationOnly": true, + "declaration": true, + "declarationMap": true + }, + "include": [ + "common/**/*", + "server/**/*", + "../../../typings/*" + ], + "references": [ + { "path": "../../../src/core/tsconfig.json" }, + { "path": "../../../src/plugins/telemetry_collection_manager/tsconfig.json" }, + { "path": "../../../src/plugins/telemetry/tsconfig.json" } + ] +} + diff --git a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/step_define_form.tsx b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/step_define_form.tsx index 8f19db6fc827da..404f9aeec7c685 100644 --- a/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/step_define_form.tsx +++ b/x-pack/plugins/transform/public/app/sections/create_transform/components/step_define/step_define_form.tsx @@ -102,11 +102,6 @@ export const StepDefineForm: FC = React.memo((props) => { toastNotifications, }; - // TODO This should use the actual value of `indices.query.bool.max_clause_count` - const maxIndexFields = 1024; - const numIndexFields = indexPattern.fields.length; - const disabledQuery = numIndexFields > maxIndexFields; - const copyToClipboardSource = getIndexDevConsoleStatement(pivotQuery, indexPattern.title); const copyToClipboardSourceDescription = i18n.translate( 'xpack.transform.indexPreview.copyClipboardTooltip', @@ -181,18 +176,6 @@ export const StepDefineForm: FC = React.memo((props) => { label={i18n.translate('xpack.transform.stepDefineForm.indexPatternLabel', { defaultMessage: 'Index pattern', })} - helpText={ - disabledQuery - ? i18n.translate('xpack.transform.stepDefineForm.indexPatternHelpText', { - defaultMessage: - 'An optional query for this index pattern is not supported. The number of supported index fields is {maxIndexFields} whereas this index has {numIndexFields} fields.', - values: { - maxIndexFields, - numIndexFields, - }, - }) - : '' - } > {indexPattern.title} @@ -214,7 +197,7 @@ export const StepDefineForm: FC = React.memo((props) => { {/* Flex Column #1: Search Bar / Advanced Search Editor */} {searchItems.savedSearch === undefined && ( <> - {!disabledQuery && !isAdvancedSourceEditorEnabled && ( + {!isAdvancedSourceEditorEnabled && ( { , element ); - return () => unmountComponentAtNode(element); + return () => { + unmountComponentAtNode(element); + }; }; diff --git a/x-pack/plugins/triggers_actions_ui/public/application/components/health_check.test.tsx b/x-pack/plugins/triggers_actions_ui/public/application/components/health_check.test.tsx index 1d908920db8b0d..a7de73c9aab295 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/components/health_check.test.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/components/health_check.test.tsx @@ -10,6 +10,7 @@ import { HealthCheck } from './health_check'; import { act } from 'react-dom/test-utils'; import { httpServiceMock } from '../../../../../../src/core/public/mocks'; +import { HealthContextProvider } from '../context/health_context'; const docLinks = { ELASTIC_WEBSITE_URL: 'elastic.co/', DOC_LINK_VERSION: 'current' }; @@ -20,9 +21,11 @@ describe('health check', () => { http.get.mockImplementationOnce(() => new Promise(() => {})); const { queryByText, container } = render( - -

{'shouldnt render'}

- + + +

{'shouldnt render'}

+ + ); await act(async () => { // wait for useEffect to run @@ -32,13 +35,33 @@ describe('health check', () => { expect(queryByText('shouldnt render')).not.toBeInTheDocument(); }); + it('renders children immediately if waitForCheck is false', async () => { + http.get.mockImplementationOnce(() => new Promise(() => {})); + + const { queryByText, container } = render( + + +

{'should render'}

+ + + ); + await act(async () => { + // wait for useEffect to run + }); + + expect(container.getElementsByClassName('euiLoadingSpinner').length).toBe(0); + expect(queryByText('should render')).toBeInTheDocument(); + }); + it('renders children if keys are enabled', async () => { http.get.mockResolvedValue({ isSufficientlySecure: true, hasPermanentEncryptionKey: true }); const { queryByText } = render( - -

{'should render'}

- + + +

{'should render'}

+ + ); await act(async () => { // wait for useEffect to run @@ -53,9 +76,11 @@ describe('health check', () => { })); const { queryAllByText } = render( - -

{'should render'}

- + + +

{'should render'}

+ + ); await act(async () => { // wait for useEffect to run @@ -81,9 +106,11 @@ describe('health check', () => { })); const { queryByText, queryByRole } = render( - -

{'should render'}

- + + +

{'should render'}

+ + ); await act(async () => { // wait for useEffect to run @@ -108,9 +135,11 @@ describe('health check', () => { })); const { queryByText } = render( - -

{'should render'}

- + + +

{'should render'}

+ + ); await act(async () => { // wait for useEffect to run diff --git a/x-pack/plugins/triggers_actions_ui/public/application/components/health_check.tsx b/x-pack/plugins/triggers_actions_ui/public/application/components/health_check.tsx index 009f5824247654..c4d0b4976266e0 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/components/health_check.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/components/health_check.tsx @@ -18,33 +18,39 @@ import { EuiEmptyPrompt, EuiCode } from '@elastic/eui'; import { AlertingFrameworkHealth } from '../../types'; import { health } from '../lib/alert_api'; import './health_check.scss'; +import { useHealthContext } from '../context/health_context'; interface Props { docLinks: Pick; http: HttpSetup; inFlyout?: boolean; + waitForCheck: boolean; } export const HealthCheck: React.FunctionComponent = ({ docLinks, http, children, + waitForCheck, inFlyout = false, }) => { + const { setLoadingHealthCheck } = useHealthContext(); const [alertingHealth, setAlertingHealth] = React.useState>(none); React.useEffect(() => { (async function () { + setLoadingHealthCheck(true); setAlertingHealth(some(await health({ http }))); + setLoadingHealthCheck(false); })(); - }, [http]); + }, [http, setLoadingHealthCheck]); const className = inFlyout ? 'alertingFlyoutHealthCheck' : 'alertingHealthCheck'; return pipe( alertingHealth, fold( - () => , + () => (waitForCheck ? : {children}), (healthCheck) => { return healthCheck?.isSufficientlySecure && healthCheck?.hasPermanentEncryptionKey ? ( {children} diff --git a/x-pack/plugins/triggers_actions_ui/public/application/context/health_context.tsx b/x-pack/plugins/triggers_actions_ui/public/application/context/health_context.tsx new file mode 100644 index 00000000000000..de27f6db761e83 --- /dev/null +++ b/x-pack/plugins/triggers_actions_ui/public/application/context/health_context.tsx @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { createContext, useCallback, useContext, useMemo, useState } from 'react'; + +export interface HealthContextValue { + loadingHealthCheck: boolean; + setLoadingHealthCheck: (loading: boolean) => void; +} + +const defaultHealthContext: HealthContextValue = { + loadingHealthCheck: false, + setLoadingHealthCheck: (loading: boolean) => { + throw new Error( + 'setLoadingHealthCheck was not initialized, set it when you invoke the context' + ); + }, +}; + +const HealthContext = createContext(defaultHealthContext); + +export const HealthContextProvider = ({ children }: { children: React.ReactNode }) => { + const [loading, setLoading] = useState(false); + + const setLoadingHealthCheck = useCallback((isLoading: boolean) => { + setLoading(isLoading); + }, []); + + const value = useMemo(() => { + return { loadingHealthCheck: loading, setLoadingHealthCheck }; + }, [loading, setLoadingHealthCheck]); + + return {children}; +}; + +export const useHealthContext = () => { + const ctx = useContext(HealthContext); + if (!ctx) { + throw new Error('HealthContext has not been set.'); + } + return ctx; +}; diff --git a/x-pack/plugins/triggers_actions_ui/public/application/home.tsx b/x-pack/plugins/triggers_actions_ui/public/application/home.tsx index eb6b1ada3ba933..f009a04d409785 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/home.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/home.tsx @@ -31,6 +31,7 @@ import { ActionsConnectorsList } from './sections/actions_connectors_list/compon import { AlertsList } from './sections/alerts_list/components/alerts_list'; import { PLUGIN } from './constants/plugin'; import { HealthCheck } from './components/health_check'; +import { HealthContextProvider } from './context/health_context'; interface MatchParams { section: Section; @@ -139,9 +140,11 @@ export const TriggersActionsUIHome: React.FunctionComponent ( - - - + + + + + )} /> )} @@ -149,9 +152,11 @@ export const TriggersActionsUIHome: React.FunctionComponent ( - - - + + + + + )} /> diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/alert_form/alert_add.tsx b/x-pack/plugins/triggers_actions_ui/public/application/sections/alert_form/alert_add.tsx index 7be7e60c2e19ce..763462ba6ebf4c 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/sections/alert_form/alert_add.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/alert_form/alert_add.tsx @@ -10,11 +10,6 @@ import { EuiTitle, EuiFlyoutHeader, EuiFlyout, - EuiFlyoutFooter, - EuiFlexGroup, - EuiFlexItem, - EuiButtonEmpty, - EuiButton, EuiFlyoutBody, EuiPortal, EuiBetaBadge, @@ -29,6 +24,8 @@ import { HealthCheck } from '../../components/health_check'; import { PLUGIN } from '../../constants/plugin'; import { ConfirmAlertSave } from './confirm_alert_save'; import { hasShowActionsCapability } from '../../lib/capabilities'; +import AlertAddFooter from './alert_add_footer'; +import { HealthContextProvider } from '../../context/health_context'; interface AlertAddProps { consumer: string; @@ -183,54 +180,37 @@ export const AlertAdd = ({ - - - + + + + + { + setIsSaving(true); + if (shouldConfirmSave) { + setIsConfirmAlertSaveModalOpen(true); + } else { + await saveAlertAndCloseFlyout(); + } + }} + onCancel={closeFlyout} /> - - - - - - {i18n.translate('xpack.triggersActionsUI.sections.alertAdd.cancelButtonLabel', { - defaultMessage: 'Cancel', - })} - - - - { - setIsSaving(true); - if (shouldConfirmSave) { - setIsConfirmAlertSaveModalOpen(true); - } else { - await saveAlertAndCloseFlyout(); - } - }} - > - - - - - - + + {isConfirmAlertSaveModalOpen && ( { diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/alert_form/alert_add_footer.tsx b/x-pack/plugins/triggers_actions_ui/public/application/sections/alert_form/alert_add_footer.tsx new file mode 100644 index 00000000000000..92e1198de84400 --- /dev/null +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/alert_form/alert_add_footer.tsx @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import React from 'react'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { + EuiFlyoutFooter, + EuiFlexGroup, + EuiFlexItem, + EuiButtonEmpty, + EuiButton, +} from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { useHealthContext } from '../../context/health_context'; + +interface AlertAddFooterProps { + isSaving: boolean; + hasErrors: boolean; + onSave: () => void; + onCancel: () => void; +} + +export const AlertAddFooter = ({ isSaving, hasErrors, onSave, onCancel }: AlertAddFooterProps) => { + const { loadingHealthCheck } = useHealthContext(); + + return ( + + + + + {i18n.translate('xpack.triggersActionsUI.sections.alertAddFooter.cancelButtonLabel', { + defaultMessage: 'Cancel', + })} + + + + + + + + + + ); +}; + +// eslint-disable-next-line import/no-default-export +export { AlertAddFooter as default }; diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/alert_form/alert_edit.tsx b/x-pack/plugins/triggers_actions_ui/public/application/sections/alert_form/alert_edit.tsx index b60aa04ee9f272..0435a4cc33cb81 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/sections/alert_form/alert_edit.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/alert_form/alert_edit.tsx @@ -28,6 +28,7 @@ import { alertReducer } from './alert_reducer'; import { updateAlert } from '../../lib/alert_api'; import { HealthCheck } from '../../components/health_check'; import { PLUGIN } from '../../constants/plugin'; +import { HealthContextProvider } from '../../context/health_context'; interface AlertEditProps { initialAlert: Alert; @@ -135,74 +136,82 @@ export const AlertEdit = ({ initialAlert, onClose }: AlertEditProps) => { - - - {hasActionsDisabled && ( - - - - - )} - - - - - - - {i18n.translate('xpack.triggersActionsUI.sections.alertEdit.cancelButtonLabel', { - defaultMessage: 'Cancel', - })} - - - - { - setIsSaving(true); - const savedAlert = await onSaveAlert(); - setIsSaving(false); - if (savedAlert) { - closeFlyout(); - if (reloadAlerts) { - reloadAlerts(); - } - } - }} - > - + + + {hasActionsDisabled && ( + + - - - - - + + + )} + + + + + + + {i18n.translate( + 'xpack.triggersActionsUI.sections.alertEdit.cancelButtonLabel', + { + defaultMessage: 'Cancel', + } + )} + + + + { + setIsSaving(true); + const savedAlert = await onSaveAlert(); + setIsSaving(false); + if (savedAlert) { + closeFlyout(); + if (reloadAlerts) { + reloadAlerts(); + } + } + }} + > + + + + + + + ); diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/alert_form/alert_form.tsx b/x-pack/plugins/triggers_actions_ui/public/application/sections/alert_form/alert_form.tsx index 8800f149c033b0..d2ca0abe566ad7 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/sections/alert_form/alert_form.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/alert_form/alert_form.tsx @@ -105,9 +105,7 @@ export const AlertForm = ({ } = alertsContext; const canShowActions = hasShowActionsCapability(capabilities); - const [alertTypeModel, setAlertTypeModel] = useState( - alert.alertTypeId ? alertTypeRegistry.get(alert.alertTypeId) : null - ); + const [alertTypeModel, setAlertTypeModel] = useState(null); const [alertTypesIndex, setAlertTypesIndex] = useState(undefined); const [alertInterval, setAlertInterval] = useState( @@ -149,6 +147,10 @@ export const AlertForm = ({ // eslint-disable-next-line react-hooks/exhaustive-deps }, []); + useEffect(() => { + setAlertTypeModel(alert.alertTypeId ? alertTypeRegistry.get(alert.alertTypeId) : null); + }, [alert, alertTypeRegistry]); + const setAlertProperty = (key: string, value: any) => { dispatch({ command: { type: 'setProperty' }, payload: { key, value } }); }; diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_list/components/action_type_filter.tsx b/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_list/components/action_type_filter.tsx index 76c13a46db1a9f..5a444cd3e9e603 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_list/components/action_type_filter.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_list/components/action_type_filter.tsx @@ -40,6 +40,7 @@ export const ActionTypeFilter: React.FunctionComponent = numActiveFilters={selectedValues.length} numFilters={selectedValues.length} onClick={() => setIsPopoverOpen(!isPopoverOpen)} + data-test-subj="actionTypeFilterButton" > = } }} checked={selectedValues.includes(item.id) ? 'on' : undefined} + data-test-subj={`actionType${item.id}FilterOption`} > {item.name} diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_list/components/alert_status_filter.tsx b/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_list/components/alert_status_filter.tsx index 87e7a82cd8f233..1bae0b885dbfa9 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_list/components/alert_status_filter.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_list/components/alert_status_filter.tsx @@ -54,6 +54,7 @@ export const AlertStatusFilter: React.FunctionComponent numActiveFilters={selectedValues.length} numFilters={selectedValues.length} onClick={() => setIsPopoverOpen(!isPopoverOpen)} + data-test-subj="alertStatusFilterButton" > } }} checked={selectedValues.includes(item) ? 'on' : undefined} + data-test-subj={`alertStatus${item}FilerOption`} > {alertsStatusesTranslationsMapping[item]} diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_list/components/alerts_list.tsx b/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_list/components/alerts_list.tsx index 1ef695f3ed1125..9eb1149cf39050 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_list/components/alerts_list.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/alerts_list/components/alerts_list.tsx @@ -437,15 +437,14 @@ export const AlertsList: React.FunctionComponent = () => { title={ } iconType="alert" + data-test-subj="alertsErrorBanner" > { - + { - + { - + - + - + { - + = ({ numActiveFilters={selectedValues.length} numFilters={selectedValues.length} onClick={() => setIsPopoverOpen(!isPopoverOpen)} + data-test-subj="alertTypeFilterButton" > = ({ } }} checked={selectedValues.includes(item.value) ? 'on' : undefined} + data-test-subj={`alertType${item.value}FilterOption`} > {item.name} diff --git a/x-pack/plugins/triggers_actions_ui/public/plugin.ts b/x-pack/plugins/triggers_actions_ui/public/plugin.ts index ef585eca1d67cf..874a380f56b5f8 100644 --- a/x-pack/plugins/triggers_actions_ui/public/plugin.ts +++ b/x-pack/plugins/triggers_actions_ui/public/plugin.ts @@ -5,10 +5,10 @@ */ import { - CoreStart, CoreSetup, - PluginInitializerContext, + CoreStart, Plugin as CorePlugin, + PluginInitializerContext, } from 'src/core/public'; import { i18n } from '@kbn/i18n'; @@ -17,10 +17,9 @@ import { registerBuiltInAlertTypes } from './application/components/builtin_aler import { ActionTypeModel, AlertTypeModel } from './types'; import { TypeRegistry } from './application/type_registry'; import { - ManagementSetup, ManagementAppMountParams, + ManagementSetup, } from '../../../../src/plugins/management/public'; -import { boot } from './application/boot'; import { ChartsPluginStart } from '../../../../src/plugins/charts/public'; import { PluginStartContract as AlertingStart } from '../../alerts/public'; import { DataPublicPluginStart } from '../../../../src/plugins/data/public'; @@ -63,11 +62,9 @@ export class Plugin private initializerContext: PluginInitializerContext; constructor(initializerContext: PluginInitializerContext) { - const actionTypeRegistry = new TypeRegistry(); - this.actionTypeRegistry = actionTypeRegistry; + this.actionTypeRegistry = new TypeRegistry(); - const alertTypeRegistry = new TypeRegistry(); - this.alertTypeRegistry = alertTypeRegistry; + this.alertTypeRegistry = new TypeRegistry(); this.initializerContext = initializerContext; } @@ -88,7 +85,10 @@ export class Plugin PluginsStart, unknown ]; - boot({ + + const { boot } = await import('./application/boot'); + + return boot({ dataPlugin: pluginsStart.data, charts: pluginsStart.charts, alerts: pluginsStart.alerts, @@ -107,7 +107,6 @@ export class Plugin actionTypeRegistry, alertTypeRegistry, }); - return () => {}; }, }); diff --git a/x-pack/plugins/uptime/server/lib/alerts/status_check.ts b/x-pack/plugins/uptime/server/lib/alerts/status_check.ts index 3fc26811d46eb1..7feb916046e3a0 100644 --- a/x-pack/plugins/uptime/server/lib/alerts/status_check.ts +++ b/x-pack/plugins/uptime/server/lib/alerts/status_check.ts @@ -7,6 +7,7 @@ import { schema } from '@kbn/config-schema'; import { i18n } from '@kbn/i18n'; import Mustache from 'mustache'; +import { ElasticsearchClient } from 'kibana/server'; import { UptimeAlertTypeFactory } from './types'; import { esKuery } from '../../../../../../src/plugins/data/server'; import { JsonObject } from '../../../../../../src/plugins/kibana_utils/common'; @@ -81,6 +82,7 @@ export const generateFilterDSL = async ( export const formatFilterString = async ( dynamicSettings: DynamicSettings, callES: ESAPICaller, + esClient: ElasticsearchClient, filters: StatusCheckFilters, search: string, libs?: UMServerLibs @@ -88,9 +90,10 @@ export const formatFilterString = async ( await generateFilterDSL( () => libs?.requests?.getIndexPattern - ? libs?.requests?.getIndexPattern({ callES, dynamicSettings }) + ? libs?.requests?.getIndexPattern({ callES, esClient, dynamicSettings }) : getUptimeIndexPattern({ callES, + esClient, dynamicSettings, }), filters, @@ -237,6 +240,7 @@ export const statusCheckAlertFactory: UptimeAlertTypeFactory = (_server, libs) = async executor( { params: rawParams, state, services: { alertInstanceFactory } }, callES, + esClient, dynamicSettings ) { const { @@ -252,7 +256,14 @@ export const statusCheckAlertFactory: UptimeAlertTypeFactory = (_server, libs) = timerange: oldVersionTimeRange, } = rawParams; - const filterString = await formatFilterString(dynamicSettings, callES, filters, search, libs); + const filterString = await formatFilterString( + dynamicSettings, + callES, + esClient, + filters, + search, + libs + ); const timerange = oldVersionTimeRange || { from: isAutoGenerated diff --git a/x-pack/plugins/uptime/server/lib/alerts/uptime_alert_wrapper.ts b/x-pack/plugins/uptime/server/lib/alerts/uptime_alert_wrapper.ts index b8a56405ca160e..390b6d347996c1 100644 --- a/x-pack/plugins/uptime/server/lib/alerts/uptime_alert_wrapper.ts +++ b/x-pack/plugins/uptime/server/lib/alerts/uptime_alert_wrapper.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { ILegacyScopedClusterClient } from 'kibana/server'; +import { ILegacyScopedClusterClient, ElasticsearchClient } from 'kibana/server'; import { AlertExecutorOptions, AlertType, AlertTypeState } from '../../../../alerts/server'; import { savedObjectsAdapter } from '../saved_objects'; import { DynamicSettings } from '../../../common/runtime_types'; @@ -13,6 +13,7 @@ export interface UptimeAlertType extends Omit Promise; } @@ -22,13 +23,13 @@ export const uptimeAlertWrapper = (uptimeAlert: UptimeAlertType) => ({ producer: 'uptime', executor: async (options: AlertExecutorOptions) => { const { - services: { callCluster: callES }, + services: { callCluster: callES, scopedClusterClient }, } = options; const dynamicSettings = await savedObjectsAdapter.getUptimeDynamicSettings( options.services.savedObjectsClient ); - return uptimeAlert.executor(options, callES, dynamicSettings); + return uptimeAlert.executor(options, callES, scopedClusterClient, dynamicSettings); }, }); diff --git a/x-pack/plugins/uptime/server/lib/requests/get_index_pattern.ts b/x-pack/plugins/uptime/server/lib/requests/get_index_pattern.ts index 1d284143a1ab0d..06846a73ed3d70 100644 --- a/x-pack/plugins/uptime/server/lib/requests/get_index_pattern.ts +++ b/x-pack/plugins/uptime/server/lib/requests/get_index_pattern.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { LegacyAPICaller, LegacyCallAPIOptions } from 'src/core/server'; +import { ElasticsearchClient } from 'kibana/server'; import { UMElasticsearchQueryFn } from '../adapters'; import { IndexPatternsFetcher, FieldDescriptor } from '../../../../../../src/plugins/data/server'; @@ -14,15 +14,10 @@ export interface IndexPatternTitleAndFields { } export const getUptimeIndexPattern: UMElasticsearchQueryFn< - {}, + { esClient: ElasticsearchClient }, IndexPatternTitleAndFields | undefined -> = async ({ callES, dynamicSettings }) => { - const callAsCurrentUser: LegacyAPICaller = async ( - endpoint: string, - clientParams: Record = {}, - options?: LegacyCallAPIOptions - ) => callES(endpoint, clientParams, options); - const indexPatternsFetcher = new IndexPatternsFetcher(callAsCurrentUser); +> = async ({ esClient, dynamicSettings }) => { + const indexPatternsFetcher = new IndexPatternsFetcher(esClient); // Since `getDynamicIndexPattern` is called in setup_request (and thus by every endpoint) // and since `getFieldsForWildcard` will throw if the specified indices don't exist, diff --git a/x-pack/plugins/uptime/server/lib/requests/get_monitor_details.ts b/x-pack/plugins/uptime/server/lib/requests/get_monitor_details.ts index fbcbc37ae0cc28..ec750f92656b2f 100644 --- a/x-pack/plugins/uptime/server/lib/requests/get_monitor_details.ts +++ b/x-pack/plugins/uptime/server/lib/requests/get_monitor_details.ts @@ -4,6 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ +import { ElasticsearchClient } from 'kibana/server'; import { ESAPICaller, UMElasticsearchQueryFn } from '../adapters'; import { MonitorDetails, MonitorError } from '../../../common/runtime_types'; import { formatFilterString } from '../alerts/status_check'; @@ -13,10 +14,12 @@ export interface GetMonitorDetailsParams { dateStart: string; dateEnd: string; alertsClient: any; + esClient: ElasticsearchClient; } const getMonitorAlerts = async ( callES: ESAPICaller, + esClient: ElasticsearchClient, dynamicSettings: any, alertsClient: any, monitorId: string @@ -67,6 +70,7 @@ const getMonitorAlerts = async ( const parsedFilters = await formatFilterString( dynamicSettings, callES, + esClient, currAlert.params.filters, currAlert.params.search ); @@ -84,7 +88,7 @@ const getMonitorAlerts = async ( export const getMonitorDetails: UMElasticsearchQueryFn< GetMonitorDetailsParams, MonitorDetails -> = async ({ callES, dynamicSettings, monitorId, dateStart, dateEnd, alertsClient }) => { +> = async ({ callES, esClient, dynamicSettings, monitorId, dateStart, dateEnd, alertsClient }) => { const queryFilters: any = [ { range: { @@ -134,7 +138,13 @@ export const getMonitorDetails: UMElasticsearchQueryFn< const monitorError: MonitorError | undefined = data?.error; const errorTimestamp: string | undefined = data?.['@timestamp']; - const monAlerts = await getMonitorAlerts(callES, dynamicSettings, alertsClient, monitorId); + const monAlerts = await getMonitorAlerts( + callES, + esClient, + dynamicSettings, + alertsClient, + monitorId + ); return { monitorId, error: monitorError, diff --git a/x-pack/plugins/uptime/server/rest_api/index_state/get_index_pattern.ts b/x-pack/plugins/uptime/server/rest_api/index_state/get_index_pattern.ts index 26715f0ff37b64..baf999158a29e4 100644 --- a/x-pack/plugins/uptime/server/rest_api/index_state/get_index_pattern.ts +++ b/x-pack/plugins/uptime/server/rest_api/index_state/get_index_pattern.ts @@ -16,7 +16,11 @@ export const createGetIndexPatternRoute: UMRestApiRouteFactory = (libs: UMServer try { return response.ok({ body: { - ...(await libs.requests.getIndexPattern({ callES, dynamicSettings })), + ...(await libs.requests.getIndexPattern({ + callES, + esClient: _context.core.elasticsearch.client.asCurrentUser, + dynamicSettings, + })), }, }); } catch (e) { diff --git a/x-pack/plugins/uptime/server/rest_api/monitors/monitors_details.ts b/x-pack/plugins/uptime/server/rest_api/monitors/monitors_details.ts index bb54effc0d57e8..8bbb4fcb5575c2 100644 --- a/x-pack/plugins/uptime/server/rest_api/monitors/monitors_details.ts +++ b/x-pack/plugins/uptime/server/rest_api/monitors/monitors_details.ts @@ -28,6 +28,7 @@ export const createGetMonitorDetailsRoute: UMRestApiRouteFactory = (libs: UMServ body: { ...(await libs.requests.getMonitorDetails({ callES, + esClient: context.core.elasticsearch.client.asCurrentUser, dynamicSettings, monitorId, dateStart, diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/actions/migrations.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/actions/migrations.ts index 1dd88832e38b7c..5992bb54c81fd7 100644 --- a/x-pack/test/alerting_api_integration/spaces_only/tests/actions/migrations.ts +++ b/x-pack/test/alerting_api_integration/spaces_only/tests/actions/migrations.ts @@ -30,24 +30,29 @@ export default function createGetTests({ getService }: FtrProviderContext) { expect(response.status).to.eql(200); expect(response.body.config).key('incidentConfiguration'); expect(response.body.config).not.key('casesConfiguration'); - expect(response.body.config.incidentConfiguration).to.eql({ - mapping: [ - { - actionType: 'overwrite', - source: 'title', - target: 'summary', - }, - { - actionType: 'overwrite', - source: 'description', - target: 'description', - }, - { - actionType: 'append', - source: 'comments', - target: 'comments', - }, - ], + expect(response.body.config).to.eql({ + apiUrl: + 'http://elastic:changeme@localhost:5620/api/_actions-FTS-external-service-simulators/jira', + incidentConfiguration: { + mapping: [ + { + actionType: 'overwrite', + source: 'title', + target: 'summary', + }, + { + actionType: 'overwrite', + source: 'description', + target: 'description', + }, + { + actionType: 'append', + source: 'comments', + target: 'comments', + }, + ], + }, + projectKey: 'CK', }); }); }); diff --git a/x-pack/test/api_integration/apis/maps/index.js b/x-pack/test/api_integration/apis/maps/index.js index 9f1fe96c5637bc..36a8fa2ec045e5 100644 --- a/x-pack/test/api_integration/apis/maps/index.js +++ b/x-pack/test/api_integration/apis/maps/index.js @@ -18,6 +18,7 @@ export default function ({ loadTestFile, getService }) { loadTestFile(require.resolve('./migrations')); loadTestFile(require.resolve('./get_tile')); loadTestFile(require.resolve('./get_grid_tile')); + loadTestFile(require.resolve('./proxy_api')); }); }); } diff --git a/x-pack/test/api_integration/apis/maps/proxy_api.js b/x-pack/test/api_integration/apis/maps/proxy_api.js new file mode 100644 index 00000000000000..f85de9dc1670ac --- /dev/null +++ b/x-pack/test/api_integration/apis/maps/proxy_api.js @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import expect from '@kbn/expect'; + +export default function ({ getService }) { + const supertest = getService('supertest'); + + describe('EMS proxy', () => { + it('should correctly rewrite url and format', async () => { + const resp = await supertest + .get(`/api/maps/ems/files/v7.10/manifest`) + .set('kbn-xsrf', 'kibana') + .expect(200); + + expect(resp.body.layers.length).to.be.greaterThan(0); + + //Check world-layer + const worldLayer = resp.body.layers.find((layer) => layer.layer_id === 'world_countries'); + expect(worldLayer.formats.length).to.be.greaterThan(0); + expect(worldLayer.formats[0].type).to.be('geojson'); + expect(worldLayer.formats[0].url).to.be('file?id=world_countries'); + }); + }); +} diff --git a/x-pack/test/api_integration/config.ts b/x-pack/test/api_integration/config.ts index 97fd968ce7992b..fc021ffa0ca5b7 100644 --- a/x-pack/test/api_integration/config.ts +++ b/x-pack/test/api_integration/config.ts @@ -25,6 +25,7 @@ export async function getApiIntegrationConfig({ readConfigFile }: FtrConfigProvi ...xPackFunctionalTestsConfig.get('kbnTestServer'), serverArgs: [ ...xPackFunctionalTestsConfig.get('kbnTestServer.serverArgs'), + '--map.proxyElasticMapsServiceInMaps=true', '--xpack.security.session.idleTimeout=3600000', // 1 hour '--telemetry.optIn=true', '--xpack.fleet.enabled=true', diff --git a/x-pack/test/apm_api_integration/basic/tests/correlations/slow_durations.ts b/x-pack/test/apm_api_integration/basic/tests/correlations/slow_durations.ts index ada67bbec070b5..085a81c5f1bf4b 100644 --- a/x-pack/test/apm_api_integration/basic/tests/correlations/slow_durations.ts +++ b/x-pack/test/apm_api_integration/basic/tests/correlations/slow_durations.ts @@ -25,7 +25,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { 'user.username,user.id,host.ip,user_agent.name,kubernetes.pod.uuid,url.domain,container.id,service.node.name'; // Failing: See https://github.com/elastic/kibana/issues/81264 - describe.skip('Slow durations', () => { + describe('Slow durations', () => { const url = format({ pathname: `/api/apm/correlations/slow_durations`, query: { start, end, durationPercentile, fieldNames }, @@ -40,21 +40,21 @@ export default function ApiTest({ getService }: FtrProviderContext) { }); }); - describe('with default scoring', () => { - let response: PromiseReturnType; - before(async () => { - await esArchiver.load(archiveName); - response = await supertest.get(url); - }); - + describe('when data is loaded', () => { + before(() => esArchiver.load(archiveName)); after(() => esArchiver.unload(archiveName)); - it('returns successfully', () => { - expect(response.status).to.eql(200); - }); + describe('making request with default args', () => { + let response: PromiseReturnType; + before(async () => { + response = await supertest.get(url); + + it('returns successfully', () => { + expect(response.status).to.eql(200); + }); - it('returns fields in response', () => { - expectSnapshot(Object.keys(response.body.response)).toMatchInline(` + it('returns fields in response', () => { + expectSnapshot(Object.keys(response.body.response)).toMatchInline(` Array [ "service.node.name", "host.ip", @@ -64,14 +64,14 @@ export default function ApiTest({ getService }: FtrProviderContext) { "url.domain", ] `); - }); + }); - it('returns cardinality for each field', () => { - const cardinalitys = Object.values(response.body.response).map( - (field: any) => field.cardinality - ); + it('returns cardinality for each field', () => { + const cardinalitys = Object.values(response.body.response).map( + (field: any) => field.cardinality + ); - expectSnapshot(cardinalitys).toMatchInline(` + expectSnapshot(cardinalitys).toMatchInline(` Array [ 5, 6, @@ -81,11 +81,11 @@ export default function ApiTest({ getService }: FtrProviderContext) { 4, ] `); - }); + }); - it('returns buckets', () => { - const { buckets } = response.body.response['user.id'].value; - expectSnapshot(buckets[0]).toMatchInline(` + it('returns buckets', () => { + const { buckets } = response.body.response['user.id'].value; + expectSnapshot(buckets[0]).toMatchInline(` Object { "bg_count": 32, "doc_count": 6, @@ -93,46 +93,23 @@ export default function ApiTest({ getService }: FtrProviderContext) { "score": 0.1875, } `); + }); + }); }); - }); - describe('with different scoring', () => { - before(async () => esArchiver.load(archiveName)); - after(() => esArchiver.unload(archiveName)); - - it(`returns buckets for each score`, async () => { - const promises = ['percentage', 'jlh', 'chi_square', 'gnd'].map(async (scoring) => { - const response = await supertest.get( - format({ - pathname: `/api/apm/correlations/slow_durations`, - query: { start, end, durationPercentile, fieldNames, scoring }, - }) - ); - - return { name: scoring, value: response.body.response['user.id'].value.buckets[0].score }; + describe('making a request for each "scoring"', () => { + ['percentage', 'jlh', 'chi_square', 'gnd'].map(async (scoring) => { + it(`returns response for scoring "${scoring}"`, async () => { + const response = await supertest.get( + format({ + pathname: `/api/apm/correlations/slow_durations`, + query: { start, end, durationPercentile, fieldNames, scoring }, + }) + ); + + expect(response.status).to.be(200); + }); }); - - const res = await Promise.all(promises); - expectSnapshot(res).toMatchInline(` - Array [ - Object { - "name": "percentage", - "value": 0.1875, - }, - Object { - "name": "jlh", - "value": 3.33506905769659, - }, - Object { - "name": "chi_square", - "value": 219.192006524483, - }, - Object { - "name": "gnd", - "value": 0.671406580688819, - }, - ] - `); }); }); }); diff --git a/x-pack/test/functional/apps/dashboard/async_search/async_search.ts b/x-pack/test/functional/apps/dashboard/async_search/async_search.ts index 004e51e459caa5..6932a88635a679 100644 --- a/x-pack/test/functional/apps/dashboard/async_search/async_search.ts +++ b/x-pack/test/functional/apps/dashboard/async_search/async_search.ts @@ -8,10 +8,20 @@ import expect from '@kbn/expect'; import { FtrProviderContext } from '../../../ftr_provider_context'; export default function ({ getService, getPageObjects }: FtrProviderContext) { + const es = getService('es'); const testSubjects = getService('testSubjects'); + const log = getService('log'); const PageObjects = getPageObjects(['common', 'header', 'dashboard', 'visChart']); describe('dashboard with async search', () => { + before(async function () { + const { body } = await es.info(); + if (!body.version.number.includes('SNAPSHOT')) { + log.debug('Skipping because this build does not have the required shard_delay agg'); + this.skip(); + } + }); + it('not delayed should load', async () => { await PageObjects.common.navigateToApp('dashboard'); await PageObjects.dashboard.gotoDashboardEditMode('Not Delayed'); diff --git a/x-pack/test/functional/apps/lens/persistent_context.ts b/x-pack/test/functional/apps/lens/persistent_context.ts index 15515df92a89fd..8d536aac3f795b 100644 --- a/x-pack/test/functional/apps/lens/persistent_context.ts +++ b/x-pack/test/functional/apps/lens/persistent_context.ts @@ -66,5 +66,28 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { expect(timeRange.end).to.equal('Sep 19, 2025 @ 06:31:44.000'); await filterBar.hasFilter('ip', '97.220.3.248', false, true); }); + + it('keeps time range and pinned filters after refreshing directly after saving', async () => { + // restore defaults so visualization becomes saveable + await security.testUser.restoreDefaults(); + await PageObjects.lens.configureDimension({ + dimension: 'lnsXY_xDimensionPanel > lns-empty-dimension', + operation: 'date_histogram', + field: '@timestamp', + }); + + await PageObjects.lens.configureDimension({ + dimension: 'lnsXY_yDimensionPanel > lns-empty-dimension', + operation: 'avg', + field: 'bytes', + }); + await PageObjects.lens.save('persistentcontext'); + await browser.refresh(); + await PageObjects.header.waitUntilLoadingHasFinished(); + const timeRange = await PageObjects.timePicker.getTimeConfig(); + expect(timeRange.start).to.equal('Sep 7, 2015 @ 06:31:44.000'); + expect(timeRange.end).to.equal('Sep 19, 2025 @ 06:31:44.000'); + await filterBar.hasFilter('ip', '97.220.3.248', false, true); + }); }); } diff --git a/x-pack/test/functional/apps/maps/discover.js b/x-pack/test/functional/apps/maps/discover.js index 8dbd98ed3af2fa..6a2c1f8437698b 100644 --- a/x-pack/test/functional/apps/maps/discover.js +++ b/x-pack/test/functional/apps/maps/discover.js @@ -36,6 +36,7 @@ export default function ({ getService, getPageObjects }) { expect(doesLayerExist).to.equal(true); const hits = await PageObjects.maps.getHits(); expect(hits).to.equal('4'); + await PageObjects.maps.refreshAndClearUnsavedChangesWarning(); }); it('should link geo_point fields to Maps application with time and query context', async () => { @@ -55,6 +56,7 @@ export default function ({ getService, getPageObjects }) { expect(doesLayerExist).to.equal(true); const hits = await PageObjects.maps.getHits(); expect(hits).to.equal('7'); + await PageObjects.maps.refreshAndClearUnsavedChangesWarning(); }); }); } diff --git a/x-pack/test/functional/apps/maps/embeddable/dashboard.js b/x-pack/test/functional/apps/maps/embeddable/dashboard.js index 3c935b385cb39f..0c8a208e92ece9 100644 --- a/x-pack/test/functional/apps/maps/embeddable/dashboard.js +++ b/x-pack/test/functional/apps/maps/embeddable/dashboard.js @@ -15,6 +15,7 @@ export default function ({ getPageObjects, getService }) { const inspector = getService('inspector'); const testSubjects = getService('testSubjects'); const browser = getService('browser'); + const retry = getService('retry'); describe('embed in dashboard', () => { before(async () => { @@ -50,9 +51,11 @@ export default function ({ getPageObjects, getService }) { it('should populate inspector with requests for map embeddable', async () => { await dashboardPanelActions.openInspectorByTitle('join example'); - const joinExampleRequestNames = await inspector.getRequestNames(); + await retry.try(async () => { + const joinExampleRequestNames = await inspector.getRequestNames(); + expect(joinExampleRequestNames).to.equal('geo_shapes*,meta_for_geo_shapes*.shape_name'); + }); await inspector.close(); - expect(joinExampleRequestNames).to.equal('geo_shapes*,meta_for_geo_shapes*.shape_name'); await dashboardPanelActions.openInspectorByTitle('geo grid vector grid example'); const gridExampleRequestNames = await inspector.getRequestNames(); diff --git a/x-pack/test/functional/apps/maps/joins.js b/x-pack/test/functional/apps/maps/joins.js index bd5ecfe2a25049..0e2850dafbcccf 100644 --- a/x-pack/test/functional/apps/maps/joins.js +++ b/x-pack/test/functional/apps/maps/joins.js @@ -38,6 +38,7 @@ export default function ({ getPageObjects, getService }) { after(async () => { await inspector.close(); + await PageObjects.maps.refreshAndClearUnsavedChangesWarning(); await security.testUser.restoreDefaults(); }); diff --git a/x-pack/test/functional/apps/maps/layer_visibility.js b/x-pack/test/functional/apps/maps/layer_visibility.js index dd9b93c995695b..75a0e7da0f256c 100644 --- a/x-pack/test/functional/apps/maps/layer_visibility.js +++ b/x-pack/test/functional/apps/maps/layer_visibility.js @@ -19,6 +19,7 @@ export default function ({ getPageObjects, getService }) { afterEach(async () => { await inspector.close(); + await PageObjects.maps.refreshAndClearUnsavedChangesWarning(); await security.testUser.restoreDefaults(); }); diff --git a/x-pack/test/functional/apps/maps/vector_styling.js b/x-pack/test/functional/apps/maps/vector_styling.js index 1def542982dd8e..e4c5eaf892c769 100644 --- a/x-pack/test/functional/apps/maps/vector_styling.js +++ b/x-pack/test/functional/apps/maps/vector_styling.js @@ -16,6 +16,7 @@ export default function ({ getService, getPageObjects }) { await PageObjects.maps.loadSavedMap('document example'); }); after(async () => { + await PageObjects.maps.refreshAndClearUnsavedChangesWarning(); await security.testUser.restoreDefaults(); }); diff --git a/x-pack/test/functional/apps/monitoring/elasticsearch/node_detail.js b/x-pack/test/functional/apps/monitoring/elasticsearch/node_detail.js index c304b177cb04b5..a58f66214e772b 100644 --- a/x-pack/test/functional/apps/monitoring/elasticsearch/node_detail.js +++ b/x-pack/test/functional/apps/monitoring/elasticsearch/node_detail.js @@ -13,8 +13,7 @@ export default function ({ getService, getPageObjects }) { const nodesList = getService('monitoringElasticsearchNodes'); const nodeDetail = getService('monitoringElasticsearchNodeDetail'); - // Failing: See https://github.com/elastic/kibana/issues/81166 - describe.skip('Elasticsearch node detail', () => { + describe('Elasticsearch node detail', () => { describe('Active Nodes', () => { const { setup, tearDown } = getLifecycleMethods(getService, getPageObjects); diff --git a/x-pack/test/functional/page_objects/gis_page.ts b/x-pack/test/functional/page_objects/gis_page.ts index 7be0aa425509e7..c4f1bd7dc2a6b5 100644 --- a/x-pack/test/functional/page_objects/gis_page.ts +++ b/x-pack/test/functional/page_objects/gis_page.ts @@ -19,6 +19,14 @@ export function GisPageProvider({ getService, getPageObjects }: FtrProviderConte const queryBar = getService('queryBar'); const comboBox = getService('comboBox'); const renderable = getService('renderable'); + const browser = getService('browser'); + const MenuToggle = getService('MenuToggle'); + + const setViewPopoverToggle = new MenuToggle({ + name: 'SetView Popover', + menuTestSubject: 'mapSetViewForm', + toggleButtonTestSubject: 'toggleSetViewVisibilityButton', + }); function escapeLayerName(layerName: string) { return layerName.split(' ').join('_'); @@ -233,41 +241,11 @@ export function GisPageProvider({ getService, getPageObjects }: FtrProviderConte return buttons.length; } - async isSetViewPopoverOpen() { - return await testSubjects.exists('mapSetViewForm', { timeout: 500 }); - } - - async openSetViewPopover() { - const isOpen = await this.isSetViewPopoverOpen(); - if (!isOpen) { - await retry.try(async () => { - await testSubjects.click('toggleSetViewVisibilityButton'); - const isOpenAfterClick = await this.isSetViewPopoverOpen(); - if (!isOpenAfterClick) { - throw new Error('set view popover not opened'); - } - }); - } - } - - async closeSetViewPopover() { - const isOpen = await this.isSetViewPopoverOpen(); - if (isOpen) { - await retry.try(async () => { - await testSubjects.click('toggleSetViewVisibilityButton'); - const isOpenAfterClick = await this.isSetViewPopoverOpen(); - if (isOpenAfterClick) { - throw new Error('set view popover not closed'); - } - }); - } - } - async setView(lat: number, lon: number, zoom: number) { log.debug( `Set view lat: ${lat.toString()}, lon: ${lon.toString()}, zoom: ${zoom.toString()}` ); - await this.openSetViewPopover(); + await setViewPopoverToggle.open(); await testSubjects.setValue('latitudeInput', lat.toString()); await testSubjects.setValue('longitudeInput', lon.toString()); await testSubjects.setValue('zoomInput', zoom.toString()); @@ -277,11 +255,20 @@ export function GisPageProvider({ getService, getPageObjects }: FtrProviderConte async getView() { log.debug('Get view'); - await this.openSetViewPopover(); - const lat = await testSubjects.getAttribute('latitudeInput', 'value'); - const lon = await testSubjects.getAttribute('longitudeInput', 'value'); - const zoom = await testSubjects.getAttribute('zoomInput', 'value'); - await this.closeSetViewPopover(); + await setViewPopoverToggle.open(); + // this method is regularly called within a retry, so we need to reduce the timeouts + // of the retries done within the getAttribute method in order to ensure that they fail + // early enough to retry getView() + const getAttributeOptions = { + tryTimeout: 5000, + findTimeout: 1000, + }; + + const lat = await testSubjects.getAttribute('latitudeInput', 'value', getAttributeOptions); + const lon = await testSubjects.getAttribute('longitudeInput', 'value', getAttributeOptions); + const zoom = await testSubjects.getAttribute('zoomInput', 'value', getAttributeOptions); + + await setViewPopoverToggle.close(); return { lat: parseFloat(lat), lon: parseFloat(lon), @@ -692,6 +679,13 @@ export function GisPageProvider({ getService, getPageObjects }: FtrProviderConte } await testSubjects.click('mapSettingSubmitButton'); } + + async refreshAndClearUnsavedChangesWarning() { + await browser.refresh(); + // accept alert if it pops up + const alert = await browser.getAlert(); + await alert?.accept(); + } } return new GisPage(); } diff --git a/x-pack/test/functional_with_es_ssl/apps/triggers_actions_ui/alert_create_flyout.ts b/x-pack/test/functional_with_es_ssl/apps/triggers_actions_ui/alert_create_flyout.ts new file mode 100644 index 00000000000000..7d99d3635106dd --- /dev/null +++ b/x-pack/test/functional_with_es_ssl/apps/triggers_actions_ui/alert_create_flyout.ts @@ -0,0 +1,142 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import uuid from 'uuid'; +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../ftr_provider_context'; + +function generateUniqueKey() { + return uuid.v4().replace(/-/g, ''); +} + +export default ({ getPageObjects, getService }: FtrProviderContext) => { + const testSubjects = getService('testSubjects'); + const pageObjects = getPageObjects(['common', 'triggersActionsUI', 'header']); + const supertest = getService('supertest'); + const find = getService('find'); + const retry = getService('retry'); + + async function getAlertsByName(name: string) { + const { + body: { data: alerts }, + } = await supertest.get(`/api/alerts/_find?search=${name}&search_fields=name`).expect(200); + + return alerts; + } + + async function deleteAlerts(alertIds: string[]) { + alertIds.forEach(async (alertId: string) => { + await supertest.delete(`/api/alerts/alert/${alertId}`).set('kbn-xsrf', 'foo').expect(204, ''); + }); + } + + async function defineAlert(alertName: string) { + await pageObjects.triggersActionsUI.clickCreateAlertButton(); + await testSubjects.setValue('alertNameInput', alertName); + await testSubjects.click('.index-threshold-SelectOption'); + await testSubjects.click('selectIndexExpression'); + const comboBox = await find.byCssSelector('#indexSelectSearchBox'); + await comboBox.click(); + await comboBox.type('k'); + const filterSelectItem = await find.byCssSelector(`.euiFilterSelectItem`); + await filterSelectItem.click(); + await testSubjects.click('thresholdAlertTimeFieldSelect'); + await retry.try(async () => { + const fieldOptions = await find.allByCssSelector('#thresholdTimeField option'); + expect(fieldOptions[1]).not.to.be(undefined); + await fieldOptions[1].click(); + }); + await testSubjects.click('closePopover'); + // need this two out of popup clicks to close them + const nameInput = await testSubjects.find('alertNameInput'); + await nameInput.click(); + } + + describe('create alert', function () { + before(async () => { + await pageObjects.common.navigateToApp('triggersActions'); + await testSubjects.click('alertsTab'); + }); + + it('should create an alert', async () => { + const alertName = generateUniqueKey(); + await defineAlert(alertName); + + await testSubjects.click('.slack-ActionTypeSelectOption'); + await testSubjects.click('addNewActionConnectorButton-.slack'); + const slackConnectorName = generateUniqueKey(); + await testSubjects.setValue('nameInput', slackConnectorName); + await testSubjects.setValue('slackWebhookUrlInput', 'https://test'); + await find.clickByCssSelector('[data-test-subj="saveActionButtonModal"]:not(disabled)'); + const createdConnectorToastTitle = await pageObjects.common.closeToast(); + expect(createdConnectorToastTitle).to.eql(`Created '${slackConnectorName}'`); + await testSubjects.setValue('messageTextArea', 'test message '); + await testSubjects.click('messageAddVariableButton'); + await testSubjects.click('variableMenuButton-0'); + const messageTextArea = await find.byCssSelector('[data-test-subj="messageTextArea"]'); + expect(await messageTextArea.getAttribute('value')).to.eql('test message {{alertId}}'); + await messageTextArea.type(' some additional text '); + + await testSubjects.click('messageAddVariableButton'); + await testSubjects.click('variableMenuButton-1'); + + expect(await messageTextArea.getAttribute('value')).to.eql( + 'test message {{alertId}} some additional text {{alertInstanceId}}' + ); + + await testSubjects.click('saveAlertButton'); + const toastTitle = await pageObjects.common.closeToast(); + expect(toastTitle).to.eql(`Created alert "${alertName}"`); + await pageObjects.triggersActionsUI.searchAlerts(alertName); + const searchResultsAfterSave = await pageObjects.triggersActionsUI.getAlertsList(); + expect(searchResultsAfterSave).to.eql([ + { + name: alertName, + tagsText: '', + alertType: 'Index threshold', + interval: '1m', + }, + ]); + + // clean up created alert + const alertsToDelete = await getAlertsByName(alertName); + await deleteAlerts(alertsToDelete.map((alertItem: { id: string }) => alertItem.id)); + }); + + it('should show save confirmation before creating alert with no actions', async () => { + const alertName = generateUniqueKey(); + await defineAlert(alertName); + + await testSubjects.click('saveAlertButton'); + await testSubjects.existOrFail('confirmAlertSaveModal'); + await testSubjects.click('confirmAlertSaveModal > confirmModalCancelButton'); + await testSubjects.missingOrFail('confirmAlertSaveModal'); + await find.existsByCssSelector('[data-test-subj="saveAlertButton"]:not(disabled)'); + + await testSubjects.click('saveAlertButton'); + await testSubjects.existOrFail('confirmAlertSaveModal'); + await testSubjects.click('confirmAlertSaveModal > confirmModalConfirmButton'); + await testSubjects.missingOrFail('confirmAlertSaveModal'); + + const toastTitle = await pageObjects.common.closeToast(); + expect(toastTitle).to.eql(`Created alert "${alertName}"`); + await pageObjects.triggersActionsUI.searchAlerts(alertName); + const searchResultsAfterSave = await pageObjects.triggersActionsUI.getAlertsList(); + expect(searchResultsAfterSave).to.eql([ + { + name: alertName, + tagsText: '', + alertType: 'Index threshold', + interval: '1m', + }, + ]); + + // clean up created alert + const alertsToDelete = await getAlertsByName(alertName); + await deleteAlerts(alertsToDelete.map((alertItem: { id: string }) => alertItem.id)); + }); + }); +}; diff --git a/x-pack/test/functional_with_es_ssl/apps/triggers_actions_ui/alerts.ts b/x-pack/test/functional_with_es_ssl/apps/triggers_actions_ui/alerts_list.ts similarity index 66% rename from x-pack/test/functional_with_es_ssl/apps/triggers_actions_ui/alerts.ts rename to x-pack/test/functional_with_es_ssl/apps/triggers_actions_ui/alerts_list.ts index ce821b56d3a8ae..a69db68c7d35ea 100644 --- a/x-pack/test/functional_with_es_ssl/apps/triggers_actions_ui/alerts.ts +++ b/x-pack/test/functional_with_es_ssl/apps/triggers_actions_ui/alerts_list.ts @@ -14,12 +14,19 @@ function generateUniqueKey() { } export default ({ getPageObjects, getService }: FtrProviderContext) => { + const alerting = getService('alerting'); const testSubjects = getService('testSubjects'); + const find = getService('find'); const pageObjects = getPageObjects(['common', 'triggersActionsUI', 'header']); const supertest = getService('supertest'); - const find = getService('find'); const retry = getService('retry'); + async function deleteAlerts(alertIds: string[]) { + alertIds.forEach(async (alertId: string) => { + await supertest.delete(`/api/alerts/alert/${alertId}`).set('kbn-xsrf', 'foo').expect(204, ''); + }); + } + async function createAlert(overwrites: Record = {}) { const { body: createdAlert } = await supertest .post(`/api/alerts/alert`) @@ -40,132 +47,42 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { return createdAlert; } - async function getAlertsByName(name: string) { - const { - body: { data: alerts }, - } = await supertest.get(`/api/alerts/_find?search=${name}&search_fields=name`).expect(200); - - return alerts; - } - - async function deleteAlerts(alertIds: string[]) { - alertIds.forEach(async (alertId: string) => { - await supertest.delete(`/api/alerts/alert/${alertId}`).set('kbn-xsrf', 'foo').expect(204, ''); - }); + async function createFailingAlert(overwrites: Record = {}) { + const { body: createdAlert } = await supertest + .post(`/api/alerts/alert`) + .set('kbn-xsrf', 'foo') + .send({ + enabled: true, + name: generateUniqueKey(), + tags: ['foo', 'bar'], + alertTypeId: 'test.failing', + consumer: 'alerts', + schedule: { interval: '30s' }, + throttle: '1m', + actions: [], + params: {}, + ...overwrites, + }) + .expect(200); + return createdAlert; } - async function defineAlert(alertName: string) { - await pageObjects.triggersActionsUI.clickCreateAlertButton(); - await testSubjects.setValue('alertNameInput', alertName); - await testSubjects.click('.index-threshold-SelectOption'); - await testSubjects.click('selectIndexExpression'); - const comboBox = await find.byCssSelector('#indexSelectSearchBox'); - await comboBox.click(); - await comboBox.type('k'); - const filterSelectItem = await find.byCssSelector(`.euiFilterSelectItem`); - await filterSelectItem.click(); - await testSubjects.click('thresholdAlertTimeFieldSelect'); - await retry.try(async () => { - const fieldOptions = await find.allByCssSelector('#thresholdTimeField option'); - expect(fieldOptions[1]).not.to.be(undefined); - await fieldOptions[1].click(); - }); - await testSubjects.click('closePopover'); - // need this two out of popup clicks to close them - const nameInput = await testSubjects.find('alertNameInput'); - await nameInput.click(); + async function refreshAlertsList() { + await testSubjects.click('alertsTab'); } - describe('alerts', function () { + describe('alerts list', function () { before(async () => { await pageObjects.common.navigateToApp('triggersActions'); await testSubjects.click('alertsTab'); }); - it('should create an alert', async () => { - const alertName = generateUniqueKey(); - await defineAlert(alertName); - - await testSubjects.click('.slack-ActionTypeSelectOption'); - await testSubjects.click('addNewActionConnectorButton-.slack'); - const slackConnectorName = generateUniqueKey(); - await testSubjects.setValue('nameInput', slackConnectorName); - await testSubjects.setValue('slackWebhookUrlInput', 'https://test'); - await find.clickByCssSelector('[data-test-subj="saveActionButtonModal"]:not(disabled)'); - const createdConnectorToastTitle = await pageObjects.common.closeToast(); - expect(createdConnectorToastTitle).to.eql(`Created '${slackConnectorName}'`); - await testSubjects.setValue('messageTextArea', 'test message '); - await testSubjects.click('messageAddVariableButton'); - await testSubjects.click('variableMenuButton-0'); - const messageTextArea = await find.byCssSelector('[data-test-subj="messageTextArea"]'); - expect(await messageTextArea.getAttribute('value')).to.eql('test message {{alertId}}'); - await messageTextArea.type(' some additional text '); - - await testSubjects.click('messageAddVariableButton'); - await testSubjects.click('variableMenuButton-1'); - - expect(await messageTextArea.getAttribute('value')).to.eql( - 'test message {{alertId}} some additional text {{alertInstanceId}}' - ); - - await testSubjects.click('saveAlertButton'); - const toastTitle = await pageObjects.common.closeToast(); - expect(toastTitle).to.eql(`Created alert "${alertName}"`); - await pageObjects.triggersActionsUI.searchAlerts(alertName); - const searchResultsAfterSave = await pageObjects.triggersActionsUI.getAlertsList(); - expect(searchResultsAfterSave).to.eql([ - { - name: alertName, - tagsText: '', - alertType: 'Index threshold', - interval: '1m', - }, - ]); - - // clean up created alert - const alertsToDelete = await getAlertsByName(alertName); - await deleteAlerts(alertsToDelete.map((alertItem: { id: string }) => alertItem.id)); - }); - - it('should show save confirmation before creating alert with no actions', async () => { - const alertName = generateUniqueKey(); - await defineAlert(alertName); - - await testSubjects.click('saveAlertButton'); - await testSubjects.existOrFail('confirmAlertSaveModal'); - await testSubjects.click('confirmAlertSaveModal > confirmModalCancelButton'); - await testSubjects.missingOrFail('confirmAlertSaveModal'); - await find.existsByCssSelector('[data-test-subj="saveAlertButton"]:not(disabled)'); - - await testSubjects.click('saveAlertButton'); - await testSubjects.existOrFail('confirmAlertSaveModal'); - await testSubjects.click('confirmAlertSaveModal > confirmModalConfirmButton'); - await testSubjects.missingOrFail('confirmAlertSaveModal'); - - const toastTitle = await pageObjects.common.closeToast(); - expect(toastTitle).to.eql(`Created alert "${alertName}"`); - await pageObjects.triggersActionsUI.searchAlerts(alertName); - const searchResultsAfterSave = await pageObjects.triggersActionsUI.getAlertsList(); - expect(searchResultsAfterSave).to.eql([ - { - name: alertName, - tagsText: '', - alertType: 'Index threshold', - interval: '1m', - }, - ]); - - // clean up created alert - const alertsToDelete = await getAlertsByName(alertName); - await deleteAlerts(alertsToDelete.map((alertItem: { id: string }) => alertItem.id)); - }); - it('should display alerts in alphabetical order', async () => { const uniqueKey = generateUniqueKey(); const a = await createAlert({ name: 'b', tags: [uniqueKey] }); const b = await createAlert({ name: 'c', tags: [uniqueKey] }); const c = await createAlert({ name: 'a', tags: [uniqueKey] }); - + await refreshAlertsList(); await pageObjects.triggersActionsUI.searchAlerts(uniqueKey); const searchResults = await pageObjects.triggersActionsUI.getAlertsList(); @@ -179,6 +96,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { it('should search for alert', async () => { const createdAlert = await createAlert(); + await refreshAlertsList(); await pageObjects.triggersActionsUI.searchAlerts(createdAlert.name); const searchResults = await pageObjects.triggersActionsUI.getAlertsList(); @@ -195,6 +113,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { it('should search for tags', async () => { const createdAlert = await createAlert(); + await refreshAlertsList(); await pageObjects.triggersActionsUI.searchAlerts(`${createdAlert.name} foo`); const searchResults = await pageObjects.triggersActionsUI.getAlertsList(); @@ -210,13 +129,17 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { }); it('should display an empty list when search did not return any alerts', async () => { + const createdAlert = await createAlert(); + await refreshAlertsList(); await pageObjects.triggersActionsUI.searchAlerts(`An Alert That For Sure Doesn't Exist!`); expect(await pageObjects.triggersActionsUI.isAlertsListDisplayed()).to.eql(true); + await deleteAlerts([createdAlert.id]); }); it('should disable single alert', async () => { const createdAlert = await createAlert(); + await refreshAlertsList(); await pageObjects.triggersActionsUI.searchAlerts(createdAlert.name); await testSubjects.click('collapsedItemActions'); @@ -235,6 +158,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { it('should re-enable single alert', async () => { const createdAlert = await createAlert(); + await refreshAlertsList(); await pageObjects.triggersActionsUI.searchAlerts(createdAlert.name); await testSubjects.click('collapsedItemActions'); @@ -259,6 +183,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { it('should mute single alert', async () => { const createdAlert = await createAlert(); + await refreshAlertsList(); await pageObjects.triggersActionsUI.searchAlerts(createdAlert.name); await testSubjects.click('collapsedItemActions'); @@ -277,6 +202,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { it('should unmute single alert', async () => { const createdAlert = await createAlert(); + await refreshAlertsList(); await pageObjects.triggersActionsUI.searchAlerts(createdAlert.name); await testSubjects.click('collapsedItemActions'); @@ -302,6 +228,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { it('should delete single alert', async () => { const firstAlert = await createAlert(); const secondAlert = await createAlert(); + await refreshAlertsList(); await pageObjects.triggersActionsUI.searchAlerts(secondAlert.name); await testSubjects.click('collapsedItemActions'); @@ -324,6 +251,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { it('should mute all selection', async () => { const createdAlert = await createAlert(); + await refreshAlertsList(); await pageObjects.triggersActionsUI.searchAlerts(createdAlert.name); await testSubjects.click(`checkboxSelectRow-${createdAlert.id}`); @@ -347,6 +275,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { it('should unmute all selection', async () => { const createdAlert = await createAlert(); + await refreshAlertsList(); await pageObjects.triggersActionsUI.searchAlerts(createdAlert.name); await testSubjects.click(`checkboxSelectRow-${createdAlert.id}`); @@ -372,6 +301,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { it('should disable all selection', async () => { const createdAlert = await createAlert(); + await refreshAlertsList(); await pageObjects.triggersActionsUI.searchAlerts(createdAlert.name); await testSubjects.click(`checkboxSelectRow-${createdAlert.id}`); @@ -395,6 +325,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { it('should enable all selection', async () => { const createdAlert = await createAlert(); + await refreshAlertsList(); await pageObjects.triggersActionsUI.searchAlerts(createdAlert.name); await testSubjects.click(`checkboxSelectRow-${createdAlert.id}`); @@ -424,7 +355,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { const createdAlertsFirstPage = await Promise.all( times(2, () => createAlert({ name: `${namePrefix}-0${count++}` })) ); - + await refreshAlertsList(); await pageObjects.triggersActionsUI.searchAlerts(namePrefix); for (const createdAlert of createdAlertsFirstPage) { @@ -447,5 +378,140 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { const searchResultsAfterDelete = await pageObjects.triggersActionsUI.getAlertsList(); expect(searchResultsAfterDelete).to.have.length(0); }); + + it('should filter alerts by the status', async () => { + const createdAlert = await createAlert(); + const failinfAlert = await createFailingAlert(); + // initialy alert get Pending status, so we need to retry refresh list logic to get the post execution statuses + await retry.try(async () => { + await refreshAlertsList(); + const refreshResults = await pageObjects.triggersActionsUI.getAlertsListWithStatus(); + expect(refreshResults.map((item) => item.status).sort()).to.eql(['Error', 'Ok']); + }); + await testSubjects.click('alertStatusFilterButton'); + await testSubjects.click('alertStatuserrorFilerOption'); // select Error status filter + await retry.try(async () => { + const filterErrorOnlyResults = await pageObjects.triggersActionsUI.getAlertsListWithStatus(); + expect(filterErrorOnlyResults).to.eql([ + { + name: failinfAlert.name, + tagsText: 'foo, bar', + alertType: 'Test: Failing', + interval: '30s', + status: 'Error', + }, + ]); + }); + + await deleteAlerts([createdAlert.id, failinfAlert.id]); + }); + + it('should display total alerts by status and error banner only when exists alerts with status error', async () => { + const createdAlert = await createAlert(); + await retry.try(async () => { + await refreshAlertsList(); + const refreshResults = await pageObjects.triggersActionsUI.getAlertsListWithStatus(); + expect(refreshResults).to.eql([ + { + name: createdAlert.name, + tagsText: 'foo, bar', + alertType: 'Test: Noop', + interval: '1m', + status: 'Ok', + }, + ]); + }); + + const alertsErrorBannerWhenNoErrors = await find.allByCssSelector( + '[data-test-subj="alertsErrorBanner"]' + ); + expect(alertsErrorBannerWhenNoErrors).to.have.length(0); + + const failingAlert = await createFailingAlert(); + await retry.try(async () => { + await refreshAlertsList(); + const alertsErrorBannerExistErrors = await find.allByCssSelector( + '[data-test-subj="alertsErrorBanner"]' + ); + expect(alertsErrorBannerExistErrors).to.have.length(1); + expect( + await ( + await alertsErrorBannerExistErrors[0].findByCssSelector('.euiCallOutHeader') + ).getVisibleText() + ).to.equal('Error found in 1 alert.'); + }); + + expect(await testSubjects.getVisibleText('totalAlertsCount')).to.be( + 'Showing: 2 of 2 alerts.' + ); + expect(await testSubjects.getVisibleText('totalActiveAlertsCount')).to.be('Active: 0'); + expect(await testSubjects.getVisibleText('totalOkAlertsCount')).to.be('Ok: 1'); + expect(await testSubjects.getVisibleText('totalErrorAlertsCount')).to.be('Error: 1'); + expect(await testSubjects.getVisibleText('totalPendingAlertsCount')).to.be('Pending: 0'); + expect(await testSubjects.getVisibleText('totalUnknownAlertsCount')).to.be('Unknown: 0'); + + await deleteAlerts([createdAlert.id, failingAlert.id]); + }); + + it('should filter alerts by the alert type', async () => { + const noopAlert = await createAlert(); + const failinfAlert = await createFailingAlert(); + await refreshAlertsList(); + await testSubjects.click('alertTypeFilterButton'); + await testSubjects.click('alertTypetest.failingFilterOption'); + + await retry.try(async () => { + const filterFailingAlertOnlyResults = await pageObjects.triggersActionsUI.getAlertsList(); + expect(filterFailingAlertOnlyResults).to.eql([ + { + name: failinfAlert.name, + tagsText: 'foo, bar', + alertType: 'Test: Failing', + interval: '30s', + }, + ]); + }); + + await deleteAlerts([noopAlert.id, failinfAlert.id]); + }); + + it('should filter alerts by the action type', async () => { + const noopAlert = await createAlert(); + const action = await alerting.actions.createAction({ + name: `slack-${Date.now()}`, + actionTypeId: '.slack', + config: {}, + secrets: { + webhookUrl: 'https://test', + }, + }); + const noopAlertWithAction = await createAlert({ + actions: [ + { + id: action.id, + actionTypeId: '.slack', + group: 'default', + params: { level: 'info', message: 'gfghfhg' }, + }, + ], + }); + await refreshAlertsList(); + await testSubjects.click('actionTypeFilterButton'); + await testSubjects.click('actionType.slackFilterOption'); + + await retry.try(async () => { + const filterWithSlackOnlyResults = await pageObjects.triggersActionsUI.getAlertsList(); + expect(filterWithSlackOnlyResults).to.eql([ + { + name: noopAlertWithAction.name, + tagsText: 'foo, bar', + alertType: 'Test: Noop', + interval: '1m', + }, + ]); + }); + + await deleteAlerts([noopAlertWithAction.id, noopAlert.id]); + }); }); }; diff --git a/x-pack/test/functional_with_es_ssl/apps/triggers_actions_ui/index.ts b/x-pack/test/functional_with_es_ssl/apps/triggers_actions_ui/index.ts index 3292184b36056f..8600cb6c852f50 100644 --- a/x-pack/test/functional_with_es_ssl/apps/triggers_actions_ui/index.ts +++ b/x-pack/test/functional_with_es_ssl/apps/triggers_actions_ui/index.ts @@ -11,7 +11,8 @@ export default ({ loadTestFile, getService }: FtrProviderContext) => { this.tags('ciGroup10'); loadTestFile(require.resolve('./home_page')); loadTestFile(require.resolve('./connectors')); - loadTestFile(require.resolve('./alerts')); + loadTestFile(require.resolve('./alerts_list')); + loadTestFile(require.resolve('./alert_create_flyout')); loadTestFile(require.resolve('./details')); }); }; diff --git a/x-pack/test/functional_with_es_ssl/fixtures/plugins/alerts/server/plugin.ts b/x-pack/test/functional_with_es_ssl/fixtures/plugins/alerts/server/plugin.ts index fd7869eac918ff..e3927f6bfffb95 100644 --- a/x-pack/test/functional_with_es_ssl/fixtures/plugins/alerts/server/plugin.ts +++ b/x-pack/test/functional_with_es_ssl/fixtures/plugins/alerts/server/plugin.ts @@ -21,16 +21,17 @@ export class AlertingFixturePlugin implements Plugin { + return getRowItemData(row, $); + }); + }, + async getAlertsListWithStatus() { + const table = await find.byCssSelector('[data-test-subj="alertsList"] table'); + const $ = await table.parseDomContent(); + return $.findTestSubjects('alert-row') + .toArray() + .map((row) => { + const rowItem = getRowItemData(row, $); return { - name: $(row) - .findTestSubject('alertsTableCell-name') - .find('.euiTableCellContent') - .text(), - tagsText: $(row) - .findTestSubject('alertsTableCell-tagsText') - .find('.euiTableCellContent') - .text(), - alertType: $(row) - .findTestSubject('alertsTableCell-alertType') - .find('.euiTableCellContent') - .text(), - interval: $(row) - .findTestSubject('alertsTableCell-interval') + ...rowItem, + status: $(row) + .findTestSubject('alertsTableCell-status') .find('.euiTableCellContent') .text(), }; diff --git a/x-pack/test/plugin_api_integration/config.ts b/x-pack/test/plugin_api_integration/config.ts index b89ed6ad550a34..30a361ea2a379b 100644 --- a/x-pack/test/plugin_api_integration/config.ts +++ b/x-pack/test/plugin_api_integration/config.ts @@ -43,6 +43,7 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) { '--xpack.eventLog.enabled=true', '--xpack.eventLog.logEntries=true', '--xpack.eventLog.indexEntries=true', + '--xpack.task_manager.monitored_aggregated_stats_refresh_rate=5000', ...plugins.map( (pluginDir) => `--plugin-path=${path.resolve(__dirname, 'plugins', pluginDir)}` ), diff --git a/x-pack/test/plugin_api_integration/plugins/sample_task_plugin/server/plugin.ts b/x-pack/test/plugin_api_integration/plugins/sample_task_plugin/server/plugin.ts index 803df6a66ea580..b5d2c98d8cbcd0 100644 --- a/x-pack/test/plugin_api_integration/plugins/sample_task_plugin/server/plugin.ts +++ b/x-pack/test/plugin_api_integration/plugins/sample_task_plugin/server/plugin.ts @@ -4,6 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ +import _ from 'lodash'; import { Plugin, CoreSetup, CoreStart } from 'src/core/server'; import { EventEmitter } from 'events'; import { Subject } from 'rxjs'; @@ -103,6 +104,29 @@ export class SampleTaskManagerFixturePlugin // fail after the first failed run maxAttempts: 1, }, + sampleRecurringTaskTimingOut: { + title: 'Sample Recurring Task that Times Out', + description: 'A sample task that times out each run.', + maxAttempts: 3, + timeout: '1s', + createTaskRunner: () => ({ + async run() { + return await new Promise((resolve) => {}); + }, + }), + }, + sampleOneTimeTaskTimingOut: { + title: 'Sample One-Time Task that Times Out', + description: 'A sample task that times out each run.', + maxAttempts: 3, + timeout: '1s', + getRetry: (attempts: number, error: object) => new Date(Date.now() + _.random(2, 5) * 1000), + createTaskRunner: () => ({ + async run() { + return await new Promise((resolve) => {}); + }, + }), + }, }); taskManager.addMiddleware({ diff --git a/x-pack/test/plugin_api_integration/test_suites/task_manager/health_route.ts b/x-pack/test/plugin_api_integration/test_suites/task_manager/health_route.ts new file mode 100644 index 00000000000000..9b02b58573673e --- /dev/null +++ b/x-pack/test/plugin_api_integration/test_suites/task_manager/health_route.ts @@ -0,0 +1,203 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import expect from '@kbn/expect'; +import url from 'url'; +import { keyBy, mapValues } from 'lodash'; +import supertestAsPromised from 'supertest-as-promised'; +import { FtrProviderContext } from '../../ftr_provider_context'; +import { ConcreteTaskInstance } from '../../../../plugins/task_manager/server'; + +interface MonitoringStats { + last_update: string; + status: string; + stats: { + configuration: { + timestamp: string; + value: Record; + }; + workload: { + timestamp: string; + value: { + count: number; + task_types: Record; + schedule: Array<[string, number]>; + overdue: number; + estimated_schedule_density: number[]; + }; + }; + runtime: { + timestamp: string; + value: { + drift: Record; + execution: { + duration: Record>; + result_frequency_percent_as_number: Record>; + }; + polling: { + last_successful_poll: string; + result_frequency_percent_as_number: Record; + }; + }; + }; + }; +} + +export default function ({ getService }: FtrProviderContext) { + const config = getService('config'); + const retry = getService('retry'); + const supertest = supertestAsPromised(url.format(config.get('servers.kibana'))); + + function getHealthRequest() { + return supertest.get('/api/task_manager/_health').set('kbn-xsrf', 'foo'); + } + + function getHealth(): Promise { + return getHealthRequest() + .expect(200) + .then((response) => response.body); + } + + function scheduleTask(task: Partial): Promise { + return supertest + .post('/api/sample_tasks/schedule') + .set('kbn-xsrf', 'xxx') + .send({ task }) + .expect(200) + .then((response: { body: ConcreteTaskInstance }) => response.body); + } + + const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + + const monitoredAggregatedStatsRefreshRate = 5000; + + describe('health', () => { + it('should return basic configuration of task manager', async () => { + const health = await getHealth(); + expect(health.status).to.eql('OK'); + expect(health.stats.configuration.value).to.eql({ + poll_interval: 3000, + max_poll_inactivity_cycles: 10, + monitored_aggregated_stats_refresh_rate: monitoredAggregatedStatsRefreshRate, + monitored_stats_running_average_window: 50, + monitored_task_execution_thresholds: { + custom: {}, + default: { + error_threshold: 90, + warn_threshold: 80, + }, + }, + request_capacity: 1000, + max_workers: 10, + }); + }); + + it('should return the task manager workload', async () => { + const health = await getHealth(); + const { + status, + stats: { workload }, + } = health; + + expect(status).to.eql('OK'); + + const sumSampleTaskInWorkload = + (workload.value.task_types as { + sampleTask?: { count: number }; + }).sampleTask?.count ?? 0; + const scheduledWorkload = (mapValues( + keyBy(workload.value.schedule as Array<[string, number]>, ([interval, count]) => interval), + ([, count]) => count + ) as unknown) as { '37m': number | undefined; '37s': number | undefined }; + + await scheduleTask({ + taskType: 'sampleTask', + schedule: { interval: '37s' }, + }); + + await scheduleTask({ + taskType: 'sampleTask', + schedule: { interval: '37m' }, + }); + + await retry.try(async () => { + // workload is configured to refresh every 5s in FTs + await delay(monitoredAggregatedStatsRefreshRate); + + const workloadAfterScheduling = (await getHealth()).stats.workload.value; + + expect( + (workloadAfterScheduling.task_types as { sampleTask: { count: number } }).sampleTask.count + ).to.eql(sumSampleTaskInWorkload + 2); + + const schedulesWorkloadAfterScheduling = (mapValues( + keyBy( + workloadAfterScheduling.schedule as Array<[string, number]>, + ([interval]) => interval + ), + ([, count]) => count + ) as unknown) as { + '37m': number; + '37s': number; + }; + expect(schedulesWorkloadAfterScheduling['37s']).to.eql(1 + (scheduledWorkload['37s'] ?? 0)); + expect(schedulesWorkloadAfterScheduling['37m']).to.eql(1 + (scheduledWorkload['37m'] ?? 0)); + }); + }); + + it('should return a breakdown of idleTasks in the task manager workload', async () => { + const { + workload: { value: workload }, + } = (await getHealth()).stats; + + expect(typeof workload.overdue).to.eql('number'); + + expect(Array.isArray(workload.estimated_schedule_density)).to.eql(true); + + // test run with the default poll_interval of 3s and a monitored_aggregated_stats_refresh_rate of 5s, + // so we expect the estimated_schedule_density to span a minute (which means 20 buckets, as 60s / 3s = 20) + expect(workload.estimated_schedule_density.length).to.eql(20); + }); + + it('should return the task manager runtime stats', async () => { + await scheduleTask({ + taskType: 'sampleTask', + schedule: { interval: '5s' }, + }); + + const { + runtime: { + value: { drift, polling, execution }, + }, + } = (await getHealth()).stats; + + expect(isNaN(Date.parse(polling.last_successful_poll as string))).to.eql(false); + expect(typeof polling.result_frequency_percent_as_number.NoTasksClaimed).to.eql('number'); + expect(typeof polling.result_frequency_percent_as_number.RanOutOfCapacity).to.eql('number'); + expect(typeof polling.result_frequency_percent_as_number.PoolFilled).to.eql('number'); + + expect(typeof drift.p50).to.eql('number'); + expect(typeof drift.p90).to.eql('number'); + expect(typeof drift.p95).to.eql('number'); + expect(typeof drift.p99).to.eql('number'); + + expect(typeof execution.duration.sampleTask.p50).to.eql('number'); + expect(typeof execution.duration.sampleTask.p90).to.eql('number'); + expect(typeof execution.duration.sampleTask.p95).to.eql('number'); + expect(typeof execution.duration.sampleTask.p99).to.eql('number'); + + expect(typeof execution.result_frequency_percent_as_number.sampleTask.Success).to.eql( + 'number' + ); + expect(typeof execution.result_frequency_percent_as_number.sampleTask.RetryScheduled).to.eql( + 'number' + ); + expect(typeof execution.result_frequency_percent_as_number.sampleTask.Failed).to.eql( + 'number' + ); + }); + }); +} diff --git a/x-pack/test/plugin_api_integration/test_suites/task_manager/index.js b/x-pack/test/plugin_api_integration/test_suites/task_manager/index.ts similarity index 58% rename from x-pack/test/plugin_api_integration/test_suites/task_manager/index.js rename to x-pack/test/plugin_api_integration/test_suites/task_manager/index.ts index 93350ad4d2c808..b542bff3a4aa93 100644 --- a/x-pack/test/plugin_api_integration/test_suites/task_manager/index.js +++ b/x-pack/test/plugin_api_integration/test_suites/task_manager/index.ts @@ -4,9 +4,12 @@ * you may not use this file except in compliance with the Elastic License. */ -export default function ({ loadTestFile }) { +import { FtrProviderContext } from '../../ftr_provider_context'; + +export default function ({ loadTestFile }: FtrProviderContext) { describe('task_manager', function taskManagerSuite() { this.tags('ciGroup2'); - loadTestFile(require.resolve('./task_manager_integration')); + loadTestFile(require.resolve('./health_route')); + loadTestFile(require.resolve('./task_management')); }); } diff --git a/x-pack/test/plugin_api_integration/test_suites/task_manager/task_manager_integration.js b/x-pack/test/plugin_api_integration/test_suites/task_manager/task_management.ts similarity index 78% rename from x-pack/test/plugin_api_integration/test_suites/task_manager/task_manager_integration.js rename to x-pack/test/plugin_api_integration/test_suites/task_manager/task_management.ts index c87a5039360b8f..1fd313c1ac4379 100644 --- a/x-pack/test/plugin_api_integration/test_suites/task_manager/task_manager_integration.js +++ b/x-pack/test/plugin_api_integration/test_suites/task_manager/task_management.ts @@ -8,19 +8,48 @@ import _ from 'lodash'; import expect from '@kbn/expect'; import url from 'url'; import supertestAsPromised from 'supertest-as-promised'; +import { FtrProviderContext } from '../../ftr_provider_context'; +import TaskManagerMapping from '../../../../plugins/task_manager/server/saved_objects/mappings.json'; +import { + DEFAULT_MAX_WORKERS, + DEFAULT_POLL_INTERVAL, +} from '../../../../plugins/task_manager/server/config'; +import { ConcreteTaskInstance } from '../../../../plugins/task_manager/server'; const { task: { properties: taskManagerIndexMapping }, -} = require('../../../../plugins/task_manager/server/saved_objects/mappings.json'); +} = TaskManagerMapping; -const { - DEFAULT_MAX_WORKERS, - DEFAULT_POLL_INTERVAL, -} = require('../../../../plugins/task_manager/server/config.ts'); +const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); -const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); +export interface RawDoc { + _id: string; + _source: any; + _type?: string; +} +export interface SearchResults { + hits: { + hits: RawDoc[]; + }; +} -export default function ({ getService }) { +type DeprecatedConcreteTaskInstance = Omit & { + interval: string; +}; + +type SerializedConcreteTaskInstance = Omit< + ConcreteTaskInstance, + 'state' | 'params' | 'scheduledAt' | 'startedAt' | 'retryAt' | 'runAt' +> & { + state: State; + params: Params; + scheduledAt: string; + startedAt: string | null; + retryAt: string | null; + runAt: string; +}; + +export default function ({ getService }: FtrProviderContext) { const es = getService('legacyEs'); const log = getService('log'); const retry = getService('retry'); @@ -53,14 +82,18 @@ export default function ({ getService }) { } }); - function currentTasks() { + function currentTasks(): Promise<{ + docs: Array>; + }> { return supertest .get('/api/sample_tasks') .expect(200) .then((response) => response.body); } - function currentTask(task) { + function currentTask( + task: string + ): Promise> { return supertest .get(`/api/sample_tasks/task/${task}`) .send({ task }) @@ -69,32 +102,30 @@ export default function ({ getService }) { } function ensureTasksIndexRefreshed() { - return supertest - .get(`/api/ensure_tasks_index_refreshed`) - .send({}) - .expect(200) - .then((response) => response.body); + return supertest.get(`/api/ensure_tasks_index_refreshed`).send({}).expect(200); } - function historyDocs(taskId) { + function historyDocs(taskId?: string): Promise { return es .search({ index: testHistoryIndex, q: taskId ? `taskId:${taskId}` : 'type:task', }) - .then((result) => result.hits.hits); + .then((result: SearchResults) => result.hits.hits); } - function scheduleTask(task) { + function scheduleTask( + task: Partial + ): Promise { return supertest .post('/api/sample_tasks/schedule') .set('kbn-xsrf', 'xxx') .send({ task }) .expect(200) - .then((response) => response.body); + .then((response: { body: SerializedConcreteTaskInstance }) => response.body); } - function runTaskNow(task) { + function runTaskNow(task: { id: string }) { return supertest .post('/api/sample_tasks/run_now') .set('kbn-xsrf', 'xxx') @@ -103,16 +134,16 @@ export default function ({ getService }) { .then((response) => response.body); } - function scheduleTaskIfNotExists(task) { + function scheduleTaskIfNotExists(task: Partial) { return supertest .post('/api/sample_tasks/ensure_scheduled') .set('kbn-xsrf', 'xxx') .send({ task }) .expect(200) - .then((response) => response.body); + .then((response: { body: ConcreteTaskInstance }) => response.body); } - function releaseTasksWaitingForEventToComplete(event) { + function releaseTasksWaitingForEventToComplete(event: string) { return supertest .post('/api/sample_tasks/event') .set('kbn-xsrf', 'xxx') @@ -120,11 +151,17 @@ export default function ({ getService }) { .expect(200); } - function getTaskById(tasks, id) { + function getTaskById( + tasks: Array>, + id: string + ) { return tasks.filter((task) => task.id === id)[0]; } - async function provideParamsToTasksWaitingForParams(taskId, data = {}) { + async function provideParamsToTasksWaitingForParams( + taskId: string, + data: Record = {} + ) { // wait for task to start running and stall on waitForParams await retry.try(async () => { const tasks = (await currentTasks()).docs; @@ -151,7 +188,7 @@ export default function ({ getService }) { await retry.try(async () => { expect((await historyDocs()).length).to.eql(1); - const [task] = (await currentTasks()).docs; + const [task] = (await currentTasks<{ count: number }>()).docs; log.debug(`Task found: ${task.id}`); log.debug(`Task status: ${task.status}`); log.debug(`Task state: ${JSON.stringify(task.state, null, 2)}`); @@ -236,7 +273,7 @@ export default function ({ getService }) { await retry.try(async () => { expect((await historyDocs(originalTask.id)).length).to.eql(1); - const [task] = (await currentTasks()).docs; + const [task] = (await currentTasks<{ count: number }>()).docs; expect(task.attempts).to.eql(0); expect(task.state.count).to.eql(count + 1); @@ -257,7 +294,7 @@ export default function ({ getService }) { await retry.try(async () => { expect((await historyDocs()).length).to.eql(1); - const [task] = (await currentTasks()).docs; + const [task] = (await currentTasks<{ count: number }>()).docs; expect(task.attempts).to.eql(0); expect(task.state.count).to.eql(1); @@ -278,7 +315,7 @@ export default function ({ getService }) { await retry.try(async () => { expect((await historyDocs()).length).to.eql(1); - const [task] = (await currentTasks()).docs; + const [task] = (await currentTasks<{ count: number }>()).docs; expect(task.attempts).to.eql(0); expect(task.state.count).to.eql(1); @@ -299,7 +336,7 @@ export default function ({ getService }) { 1 ); - const [task] = (await currentTasks()).docs.filter( + const [task] = (await currentTasks<{ count: number }>()).docs.filter( (taskDoc) => taskDoc.id === originalTask.id ); @@ -322,7 +359,7 @@ export default function ({ getService }) { .length ).to.eql(2); - const [task] = (await currentTasks()).docs.filter( + const [task] = (await currentTasks<{ count: number }>()).docs.filter( (taskDoc) => taskDoc.id === originalTask.id ); expect(task.state.count).to.eql(2); @@ -343,7 +380,7 @@ export default function ({ getService }) { const docs = await historyDocs(originalTask.id); expect(docs.length).to.eql(1); - const task = await currentTask(originalTask.id); + const task = await currentTask<{ count: number }>(originalTask.id); expect(task.state.count).to.eql(1); @@ -393,16 +430,16 @@ export default function ({ getService }) { expect(await runNowResult).to.eql({ id: originalTask.id }); await retry.try(async () => { - const task = await currentTask(originalTask.id); + const task = await currentTask<{ count: number }>(originalTask.id); expect(task.state.count).to.eql(2); }); // drain tasks, othrwise they'll keep Task Manager stalled await retry.try(async () => { await releaseTasksWaitingForEventToComplete('releaseTheOthers'); - const tasks = (await currentTasks()).docs.filter( - (task) => task.params.originalParams.waitForEvent === 'releaseTheOthers' - ); + const tasks = ( + await currentTasks<{}, { originalParams: { waitForEvent: string } }>() + ).docs.filter((task) => task.params.originalParams.waitForEvent === 'releaseTheOthers'); expect(tasks.length).to.eql(0); }); }); @@ -420,7 +457,7 @@ export default function ({ getService }) { 1 ); - const task = await currentTask(originalTask.id); + const task = await currentTask<{ count: number }>(originalTask.id); expect(task.state.count).to.eql(1); expect(task.status).to.eql('idle'); @@ -437,7 +474,7 @@ export default function ({ getService }) { expect(successfulRunNowResult).to.eql({ id: originalTask.id }); await retry.try(async () => { - const task = await currentTask(originalTask.id); + const task = await currentTask<{ count: number }>(originalTask.id); expect(task.state.count).to.eql(2); expect(task.status).to.eql('idle'); }); @@ -515,7 +552,7 @@ export default function ({ getService }) { // finish first run by emitting 'runNowHasBeenAttempted' event await releaseTasksWaitingForEventToComplete('runNowHasBeenAttempted'); await retry.try(async () => { - const tasks = (await currentTasks()).docs; + const tasks = (await currentTasks<{ count: number }>()).docs; expect(getTaskById(tasks, longRunningTask.id).state.count).to.eql(1); const task = await currentTask(longRunningTask.id); @@ -564,12 +601,14 @@ export default function ({ getService }) { expect(await runNowResultWithExpectedFailure).to.eql({ id: taskThatFailsBeforeRunNow.id }); }); - async function expectReschedule(originalRunAt, currentTask, expectedDiff) { + async function expectReschedule( + originalRunAt: number, + task: SerializedConcreteTaskInstance, + expectedDiff: number + ) { const buffer = 10000; - expect(Date.parse(currentTask.runAt) - originalRunAt).to.be.greaterThan( - expectedDiff - buffer - ); - expect(Date.parse(currentTask.runAt) - originalRunAt).to.be.lessThan(expectedDiff + buffer); + expect(Date.parse(task.runAt) - originalRunAt).to.be.greaterThan(expectedDiff - buffer); + expect(Date.parse(task.runAt) - originalRunAt).to.be.lessThan(expectedDiff + buffer); } it('should run tasks in parallel, allowing for long running tasks along side faster tasks', async () => { @@ -594,18 +633,47 @@ export default function ({ getService }) { }); await retry.try(async () => { - const tasks = (await currentTasks()).docs; + const tasks = (await currentTasks<{ count: number }>()).docs; expect(getTaskById(tasks, fastTask.id).state.count).to.eql(2); }); await releaseTasksWaitingForEventToComplete('rescheduleHasHappened'); await retry.try(async () => { - const tasks = (await currentTasks()).docs; + const tasks = (await currentTasks<{ count: number }>()).docs; expect(getTaskById(tasks, fastTask.id).state.count).to.greaterThan(2); expect(getTaskById(tasks, longRunningTask.id).state.count).to.eql(1); }); }); + + it('should mark non-recurring task as failed if task is still running but maxAttempts has been reached', async () => { + const task = await scheduleTask({ + taskType: 'sampleOneTimeTaskTimingOut', + params: {}, + }); + + await retry.try(async () => { + const [scheduledTask] = (await currentTasks()).docs; + expect(scheduledTask.id).to.eql(task.id); + expect(scheduledTask.status).to.eql('failed'); + expect(scheduledTask.attempts).to.eql(3); + }); + }); + + it('should continue claiming recurring task even if maxAttempts has been reached', async () => { + const task = await scheduleTask({ + taskType: 'sampleRecurringTaskTimingOut', + schedule: { interval: '1s' }, + params: {}, + }); + + await retry.try(async () => { + const [scheduledTask] = (await currentTasks()).docs; + expect(scheduledTask.id).to.eql(task.id); + expect(scheduledTask.status).to.eql('claiming'); + expect(scheduledTask.attempts).to.eql(4); + }); + }); }); } diff --git a/x-pack/test/tsconfig.json b/x-pack/test/tsconfig.json index 08897d08379a59..a00cf5222e512d 100644 --- a/x-pack/test/tsconfig.json +++ b/x-pack/test/tsconfig.json @@ -27,6 +27,7 @@ { "path": "../../src/plugins/telemetry/tsconfig.json" }, { "path": "../../src/plugins/kibana_usage_collection/tsconfig.json" }, { "path": "../../src/plugins/newsfeed/tsconfig.json" }, + { "path": "../plugins/telemetry_collection_xpack/tsconfig.json" }, { "path": "../plugins/task_manager/tsconfig.json"} ] } diff --git a/x-pack/tsconfig.json b/x-pack/tsconfig.json index 64d25226879274..8247b57a7a0973 100644 --- a/x-pack/tsconfig.json +++ b/x-pack/tsconfig.json @@ -15,6 +15,7 @@ "plugins/licensing/**/*", "plugins/global_search/**/*", "plugins/task_manager/**/*", + "plugins/telemetry_collection_xpack/**/*" ], "compilerOptions": { "paths": { @@ -36,6 +37,7 @@ { "path": "../src/plugins/telemetry_collection_manager/tsconfig.json" }, { "path": "../src/plugins/telemetry/tsconfig.json" }, { "path": "../src/plugins/kibana_usage_collection/tsconfig.json" }, + { "path": "./plugins/telemetry_collection_xpack/tsconfig.json" }, { "path": "../src/plugins/newsfeed/tsconfig.json" }, { "path": "./plugins/task_manager/tsconfig.json"} ] diff --git a/x-pack/tsconfig.refs.json b/x-pack/tsconfig.refs.json index 3e459cad0148e2..c2531354c39430 100644 --- a/x-pack/tsconfig.refs.json +++ b/x-pack/tsconfig.refs.json @@ -3,6 +3,7 @@ "references": [ { "path": "./plugins/licensing/tsconfig.json" }, { "path": "./plugins/global_search/tsconfig.json" }, - { "path": "./plugins/task_manager/tsconfig.json"} + { "path": "./plugins/telemetry_collection_xpack/tsconfig.json" }, + { "path": "./plugins/task_manager/tsconfig.json"}, ] }