diff --git a/packages/kbn-securitysolution-io-ts-types/src/iso_date_string/index.ts b/packages/kbn-securitysolution-io-ts-types/src/iso_date_string/index.ts index f59b07c554b654b..e34f62b1c83938f 100644 --- a/packages/kbn-securitysolution-io-ts-types/src/iso_date_string/index.ts +++ b/packages/kbn-securitysolution-io-ts-types/src/iso_date_string/index.ts @@ -13,6 +13,7 @@ import { Either } from 'fp-ts/lib/Either'; * Types the IsoDateString as: * - A string that is an ISOString */ +export type IsoDateString = t.TypeOf; export const IsoDateString = new t.Type( 'IsoDateString', t.string.is, diff --git a/x-pack/plugins/alerting/server/rules_client/lib/validate_rule_aggregation_fields.ts b/x-pack/plugins/alerting/server/rules_client/lib/validate_rule_aggregation_fields.ts index 14fd204ddc009d1..6b61d7aa5c324d2 100644 --- a/x-pack/plugins/alerting/server/rules_client/lib/validate_rule_aggregation_fields.ts +++ b/x-pack/plugins/alerting/server/rules_client/lib/validate_rule_aggregation_fields.ts @@ -17,7 +17,8 @@ const ALLOW_FIELDS = [ 'alert.attributes.snoozeSchedule.duration', 'alert.attributes.alertTypeId', 'alert.attributes.enabled', - 'alert.attributes.params.*', + 'alert.attributes.params.*', // TODO: https://github.com/elastic/kibana/issues/159602 + 'alert.attributes.params.immutable', // TODO: Remove after addressing https://github.com/elastic/kibana/issues/159602 ]; const ALLOW_AGG_TYPES = ['terms', 'composite', 'nested', 'filter']; diff --git a/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/detection_engine_health/README.md b/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/detection_engine_health/README.md new file mode 100644 index 000000000000000..77ccc448531ee08 --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/detection_engine_health/README.md @@ -0,0 +1,1710 @@ +# Detection Engine health API + +Epic: https://github.com/elastic/kibana/issues/125642 + +This health API allows users to get health overview of the Detection Engine across the whole cluster, or within a given Kibana space, or for a given rule. It can be useful for troubleshooting issues with cluster provisioning/scaling, issues with certain rules failing or generating too much load on the cluster, identifying common rule execution errors, etc. + +In the future, this API might become helpful for building more Rule Monitoring UIs giving our users more clarity and transparency about the work of the Detection Engine. + +## Rule health endpoint + +🚧 NOTE: this endpoint is **partially implemented**. 🚧 + +```txt +POST /internal/detection_engine/health/_rule +``` + +Get health overview of a rule. Scope: a given detection rule in the current Kibana space. +Returns: + +- health stats at the moment of the API call (rule and its execution summary) +- health stats over a specified period of time ("health interval") +- health stats history within the same interval in the form of a histogram + (the same stats are calculated over each of the discreet sub-intervals of the whole interval) + +Minimal required parameters: + +```json +{ + "rule_id": "d4beff10-f045-11ed-89d8-3b6931af10bc" +} +``` + +Response: + +```json +{ + "timings": { + "requested_at": "2023-05-26T16:09:54.128Z", + "processed_at": "2023-05-26T16:09:54.778Z", + "processing_time_ms": 650 + }, + "parameters": { + "interval": { + "type": "last_day", + "granularity": "hour", + "from": "2023-05-25T16:09:54.128Z", + "to": "2023-05-26T16:09:54.128Z", + "duration": "PT24H" + }, + "rule_id": "d4beff10-f045-11ed-89d8-3b6931af10bc" + }, + "health": { + "stats_at_the_moment": { + "rule": { + "id": "d4beff10-f045-11ed-89d8-3b6931af10bc", + "updated_at": "2023-05-26T15:44:21.689Z", + "updated_by": "elastic", + "created_at": "2023-05-11T21:50:23.830Z", + "created_by": "elastic", + "name": "Test rule", + "tags": ["foo"], + "interval": "1m", + "enabled": true, + "revision": 2, + "description": "-", + "risk_score": 21, + "severity": "low", + "license": "", + "output_index": "", + "meta": { + "from": "6h", + "kibana_siem_app_url": "http://localhost:5601/kbn/app/security" + }, + "author": [], + "false_positives": [], + "from": "now-21660s", + "rule_id": "e46eaaf3-6d81-4cdb-8cbb-b2201a11358b", + "max_signals": 100, + "risk_score_mapping": [], + "severity_mapping": [], + "threat": [], + "to": "now", + "references": [], + "version": 3, + "exceptions_list": [], + "immutable": false, + "related_integrations": [], + "required_fields": [], + "setup": "", + "type": "query", + "language": "kuery", + "index": [ + "apm-*-transaction*", + "auditbeat-*", + "endgame-*", + "filebeat-*", + "logs-*", + "packetbeat-*", + "traces-apm*", + "winlogbeat-*", + "-*elastic-cloud-logs-*", + "foo-*" + ], + "query": "*", + "filters": [], + "actions": [ + { + "group": "default", + "id": "bd59c4e0-f045-11ed-89d8-3b6931af10bc", + "params": { + "body": "Hello world" + }, + "action_type_id": ".webhook", + "uuid": "f8b87eb0-58bb-4d4b-a584-084d44ab847e", + "frequency": { + "summary": true, + "throttle": null, + "notifyWhen": "onActiveAlert" + } + } + ], + "execution_summary": { + "last_execution": { + "date": "2023-05-26T16:09:36.848Z", + "status": "succeeded", + "status_order": 0, + "message": "Rule execution completed successfully", + "metrics": { + "total_search_duration_ms": 2, + "execution_gap_duration_s": 80395 + } + } + } + } + }, + "stats_over_interval": { + "number_of_executions": { + "total": 21, + "by_outcome": { + "succeeded": 20, + "warning": 0, + "failed": 1 + } + }, + "number_of_logged_messages": { + "total": 42, + "by_level": { + "error": 1, + "warn": 0, + "info": 41, + "debug": 0, + "trace": 0 + } + }, + "number_of_detected_gaps": { + "total": 1, + "total_duration_s": 80395 + }, + "schedule_delay_ms": { + "percentiles": { + "1.0": 3061, + "5.0": 3083, + "25.0": 3112, + "50.0": 6049, + "75.0": 6069.5, + "95.0": 100093.79999999986, + "99.0": 207687 + } + }, + "execution_duration_ms": { + "percentiles": { + "1.0": 226, + "5.0": 228.2, + "25.0": 355.5, + "50.0": 422, + "75.0": 447, + "95.0": 677.75, + "99.0": 719 + } + }, + "search_duration_ms": { + "percentiles": { + "1.0": 0, + "5.0": 1.1, + "25.0": 2.75, + "50.0": 7, + "75.0": 13.5, + "95.0": 29.59999999999998, + "99.0": 45 + } + }, + "indexing_duration_ms": { + "percentiles": { + "1.0": 0, + "5.0": 0, + "25.0": 0, + "50.0": 0, + "75.0": 0, + "95.0": 0, + "99.0": 0 + } + }, + "top_errors": [ + { + "count": 1, + "message": "day were not queried between this rule execution and the last execution so signals may have been missed Consider increasing your look behind time or adding more Kibana instances" + } + ], + "top_warnings": [] + }, + "history_over_interval": { + "buckets": [ + { + "timestamp": "2023-05-26T15:00:00.000Z", + "stats": { + "number_of_executions": { + "total": 12, + "by_outcome": { + "succeeded": 11, + "warning": 0, + "failed": 1 + } + }, + "number_of_logged_messages": { + "total": 24, + "by_level": { + "error": 1, + "warn": 0, + "info": 23, + "debug": 0, + "trace": 0 + } + }, + "number_of_detected_gaps": { + "total": 1, + "total_duration_s": 80395 + }, + "schedule_delay_ms": { + "percentiles": { + "1.0": 3106, + "5.0": 3106.8, + "25.0": 3124.5, + "50.0": 6067.5, + "75.0": 9060.5, + "95.0": 188124.59999999971, + "99.0": 207687 + } + }, + "execution_duration_ms": { + "percentiles": { + "1.0": 230, + "5.0": 236.2, + "25.0": 354, + "50.0": 405, + "75.0": 447.5, + "95.0": 563.3999999999999, + "99.0": 576 + } + }, + "search_duration_ms": { + "percentiles": { + "1.0": 0, + "5.0": 0.20000000000000018, + "25.0": 2.5, + "50.0": 5, + "75.0": 14, + "95.0": 42.19999999999996, + "99.0": 45 + } + }, + "indexing_duration_ms": { + "percentiles": { + "1.0": 0, + "5.0": 0, + "25.0": 0, + "50.0": 0, + "75.0": 0, + "95.0": 0, + "99.0": 0 + } + } + } + }, + { + "timestamp": "2023-05-26T16:00:00.000Z", + "stats": { + "number_of_executions": { + "total": 9, + "by_outcome": { + "succeeded": 9, + "warning": 0, + "failed": 0 + } + }, + "number_of_logged_messages": { + "total": 18, + "by_level": { + "error": 0, + "warn": 0, + "info": 18, + "debug": 0, + "trace": 0 + } + }, + "number_of_detected_gaps": { + "total": 0, + "total_duration_s": 0 + }, + "schedule_delay_ms": { + "percentiles": { + "1.0": 3061, + "5.0": 3061, + "25.0": 3104.75, + "50.0": 3115, + "75.0": 6053, + "95.0": 6068, + "99.0": 6068 + } + }, + "execution_duration_ms": { + "percentiles": { + "1.0": 226.00000000000003, + "5.0": 226, + "25.0": 356, + "50.0": 436, + "75.0": 495.5, + "95.0": 719, + "99.0": 719 + } + }, + "search_duration_ms": { + "percentiles": { + "1.0": 2, + "5.0": 2, + "25.0": 5.75, + "50.0": 8, + "75.0": 13.75, + "95.0": 17, + "99.0": 17 + } + }, + "indexing_duration_ms": { + "percentiles": { + "1.0": 0, + "5.0": 0, + "25.0": 0, + "50.0": 0, + "75.0": 0, + "95.0": 0, + "99.0": 0 + } + } + } + } + ] + } + } +} +``` + +## Space health endpoint + +🚧 NOTE: this endpoint is **partially implemented**. 🚧 + +```txt +POST /internal/detection_engine/health/_space +``` + +Get health overview of the current Kibana space. Scope: all detection rules in the space. +Returns: + +- health stats at the moment of the API call +- health stats over a specified period of time ("health interval") +- health stats history within the same interval in the form of a histogram + (the same stats are calculated over each of the discreet sub-intervals of the whole interval) + +Minimal required parameters: empty object. + +```json +{} +``` + +Response: + +```json +{ + "timings": { + "requested_at": "2023-05-26T16:24:21.628Z", + "processed_at": "2023-05-26T16:24:22.880Z", + "processing_time_ms": 1252 + }, + "parameters": { + "interval": { + "type": "last_day", + "granularity": "hour", + "from": "2023-05-25T16:24:21.628Z", + "to": "2023-05-26T16:24:21.628Z", + "duration": "PT24H" + } + }, + "health": { + "stats_at_the_moment": { + "number_of_rules": { + "all": { + "total": 777, + "enabled": 777, + "disabled": 0 + }, + "by_origin": { + "prebuilt": { + "total": 776, + "enabled": 776, + "disabled": 0 + }, + "custom": { + "total": 1, + "enabled": 1, + "disabled": 0 + } + }, + "by_type": { + "siem.eqlRule": { + "total": 381, + "enabled": 381, + "disabled": 0 + }, + "siem.queryRule": { + "total": 325, + "enabled": 325, + "disabled": 0 + }, + "siem.mlRule": { + "total": 47, + "enabled": 47, + "disabled": 0 + }, + "siem.thresholdRule": { + "total": 18, + "enabled": 18, + "disabled": 0 + }, + "siem.newTermsRule": { + "total": 4, + "enabled": 4, + "disabled": 0 + }, + "siem.indicatorRule": { + "total": 2, + "enabled": 2, + "disabled": 0 + } + }, + "by_outcome": { + "warning": { + "total": 307, + "enabled": 307, + "disabled": 0 + }, + "succeeded": { + "total": 266, + "enabled": 266, + "disabled": 0 + }, + "failed": { + "total": 204, + "enabled": 204, + "disabled": 0 + } + } + } + }, + "stats_over_interval": { + "number_of_executions": { + "total": 5622, + "by_outcome": { + "succeeded": 1882, + "warning": 2129, + "failed": 2120 + } + }, + "number_of_logged_messages": { + "total": 11756, + "by_level": { + "error": 2120, + "warn": 2129, + "info": 7507, + "debug": 0, + "trace": 0 + } + }, + "number_of_detected_gaps": { + "total": 777, + "total_duration_s": 514415894 + }, + "schedule_delay_ms": { + "percentiles": { + "1.0": 216, + "5.0": 3048.5, + "25.0": 3105, + "50.0": 3129, + "75.0": 6112.355119825708, + "95.0": 134006, + "99.0": 195578 + } + }, + "execution_duration_ms": { + "percentiles": { + "1.0": 275, + "5.0": 323.375, + "25.0": 370.80555555555554, + "50.0": 413.1122337092731, + "75.0": 502.25233127864715, + "95.0": 685.8055555555555, + "99.0": 1194.75 + } + }, + "search_duration_ms": { + "percentiles": { + "1.0": 0, + "5.0": 0, + "25.0": 0, + "50.0": 0, + "75.0": 15, + "95.0": 30, + "99.0": 99.44000000000005 + } + }, + "indexing_duration_ms": { + "percentiles": { + "1.0": 0, + "5.0": 0, + "25.0": 0, + "50.0": 0, + "75.0": 0, + "95.0": 0, + "99.0": 0 + } + }, + "top_errors": [ + { + "count": 1202, + "message": "An error occurred during rule execution message verification_exception" + }, + { + "count": 777, + "message": "were not queried between this rule execution and the last execution so signals may have been missed Consider increasing your look behind time or adding more Kibana instances" + }, + { + "count": 3, + "message": "An error occurred during rule execution message rare_error_code missing" + }, + { + "count": 3, + "message": "An error occurred during rule execution message v3_windows_anomalous_path_activity missing" + }, + { + "count": 3, + "message": "An error occurred during rule execution message v3_windows_rare_user_type10_remote_login missing" + } + ], + "top_warnings": [ + { + "count": 2129, + "message": "This rule is attempting to query data from Elasticsearch indices listed in the Index pattern section of the rule definition however no index matching was found This warning will continue to appear until matching index is created or this rule is disabled" + } + ] + }, + "history_over_interval": { + "buckets": [ + { + "timestamp": "2023-05-26T15:00:00.000Z", + "stats": { + "number_of_executions": { + "total": 2245, + "by_outcome": { + "succeeded": 566, + "warning": 849, + "failed": 1336 + } + }, + "number_of_logged_messages": { + "total": 4996, + "by_level": { + "error": 1336, + "warn": 849, + "info": 2811, + "debug": 0, + "trace": 0 + } + }, + "number_of_detected_gaps": { + "total": 777, + "total_duration_s": 514415894 + }, + "schedule_delay_ms": { + "percentiles": { + "1.0": 256, + "5.0": 3086.9722222222217, + "25.0": 3133, + "50.0": 6126, + "75.0": 59484.25, + "95.0": 179817.25, + "99.0": 202613 + } + }, + "execution_duration_ms": { + "percentiles": { + "1.0": 280.6, + "5.0": 327.7, + "25.0": 371.5208333333333, + "50.0": 415.6190476190476, + "75.0": 505.7642857142857, + "95.0": 740.4375, + "99.0": 1446.1500000000005 + } + }, + "search_duration_ms": { + "percentiles": { + "1.0": 0, + "5.0": 0, + "25.0": 0, + "50.0": 0, + "75.0": 8, + "95.0": 25, + "99.0": 46 + } + }, + "indexing_duration_ms": { + "percentiles": { + "1.0": 0, + "5.0": 0, + "25.0": 0, + "50.0": 0, + "75.0": 0, + "95.0": 0, + "99.0": 0 + } + } + } + }, + { + "timestamp": "2023-05-26T16:00:00.000Z", + "stats": { + "number_of_executions": { + "total": 3363, + "by_outcome": { + "succeeded": 1316, + "warning": 1280, + "failed": 784 + } + }, + "number_of_logged_messages": { + "total": 6760, + "by_level": { + "error": 784, + "warn": 1280, + "info": 4696, + "debug": 0, + "trace": 0 + } + }, + "number_of_detected_gaps": { + "total": 0, + "total_duration_s": 0 + }, + "schedule_delay_ms": { + "percentiles": { + "1.0": 207, + "5.0": 3042, + "25.0": 3098.46511627907, + "50.0": 3112, + "75.0": 3145.2820512820517, + "95.0": 6100.571428571428, + "99.0": 6123 + } + }, + "execution_duration_ms": { + "percentiles": { + "1.0": 275, + "5.0": 319.85714285714283, + "25.0": 370.0357142857143, + "50.0": 410.79999229108853, + "75.0": 500.7692307692308, + "95.0": 675, + "99.0": 781.3999999999996 + } + }, + "search_duration_ms": { + "percentiles": { + "1.0": 0, + "5.0": 0, + "25.0": 0, + "50.0": 9, + "75.0": 17.555555555555557, + "95.0": 34, + "99.0": 110.5 + } + }, + "indexing_duration_ms": { + "percentiles": { + "1.0": 0, + "5.0": 0, + "25.0": 0, + "50.0": 0, + "75.0": 0, + "95.0": 0, + "99.0": 0 + } + } + } + } + ] + } + } +} +``` + +## Cluster health endpoint + +🚧 NOTE: this endpoint is **not implemented**. 🚧 + +```txt +POST /internal/detection_engine/health/_cluster +``` + +Minimal required parameters: empty object. + +```json +{} +``` + +Response: + +```json +{ + "message": "Not implemented", + "timings": { + "requested_at": "2023-05-26T16:32:01.878Z", + "processed_at": "2023-05-26T16:32:01.881Z", + "processing_time_ms": 3 + }, + "parameters": { + "interval": { + "type": "last_week", + "granularity": "hour", + "from": "2023-05-19T16:32:01.878Z", + "to": "2023-05-26T16:32:01.878Z", + "duration": "PT168H" + } + }, + "health": { + "stats_at_the_moment": { + "number_of_rules": { + "all": { + "total": 0, + "enabled": 0, + "disabled": 0 + }, + "by_origin": { + "prebuilt": { + "total": 0, + "enabled": 0, + "disabled": 0 + }, + "custom": { + "total": 0, + "enabled": 0, + "disabled": 0 + } + }, + "by_type": {}, + "by_outcome": {} + } + }, + "stats_over_interval": { + "message": "Not implemented" + }, + "history_over_interval": { + "buckets": [] + } + } +} +``` + +## Optional parameters + +All the three endpoints accept optional `interval` and `debug` request parameters. + +### Health interval + +You can change the interval over which the health stats will be calculated. If you don't specify it, by default health stats will be calculated over the last day with the granularity of 1 hour. + +```json +{ + "interval": { + "type": "last_week", + "granularity": "day" + } +} +``` + +You can also specify a custom date range with exact interval bounds. + +```json +{ + "interval": { + "type": "custom_range", + "granularity": "minute", + "from": "2023-05-20T16:24:21.628Z", + "to": "2023-05-26T16:24:21.628Z" + } +} +``` + +Please keep in mind that requesting large intervals with small granularity can generate substantial load on the system and enormous API responses. + +### Debug mode + +You can also include various debug information in the response, such as queries and aggregations sent to Elasticsearch and response received from it. + +```json +{ + "debug": true +} +``` + +In the response you will find something like that: + +```json +{ + "health": { + "debug": { + "rulesClient": { + "request": { + "aggs": { + "rulesByEnabled": { + "terms": { + "field": "alert.attributes.enabled" + } + }, + "rulesByOrigin": { + "terms": { + "field": "alert.attributes.params.immutable" + }, + "aggs": { + "rulesByEnabled": { + "terms": { + "field": "alert.attributes.enabled" + } + } + } + }, + "rulesByType": { + "terms": { + "field": "alert.attributes.alertTypeId" + }, + "aggs": { + "rulesByEnabled": { + "terms": { + "field": "alert.attributes.enabled" + } + } + } + }, + "rulesByOutcome": { + "terms": { + "field": "alert.attributes.lastRun.outcome" + }, + "aggs": { + "rulesByEnabled": { + "terms": { + "field": "alert.attributes.enabled" + } + } + } + } + } + }, + "response": { + "aggregations": { + "rulesByOutcome": { + "doc_count_error_upper_bound": 0, + "sum_other_doc_count": 0, + "buckets": [ + { + "key": "warning", + "doc_count": 307, + "rulesByEnabled": { + "doc_count_error_upper_bound": 0, + "sum_other_doc_count": 0, + "buckets": [ + { + "key": 1, + "key_as_string": "true", + "doc_count": 307 + } + ] + } + }, + { + "key": "succeeded", + "doc_count": 266, + "rulesByEnabled": { + "doc_count_error_upper_bound": 0, + "sum_other_doc_count": 0, + "buckets": [ + { + "key": 1, + "key_as_string": "true", + "doc_count": 266 + } + ] + } + }, + { + "key": "failed", + "doc_count": 204, + "rulesByEnabled": { + "doc_count_error_upper_bound": 0, + "sum_other_doc_count": 0, + "buckets": [ + { + "key": 1, + "key_as_string": "true", + "doc_count": 204 + } + ] + } + } + ] + }, + "rulesByType": { + "doc_count_error_upper_bound": 0, + "sum_other_doc_count": 0, + "buckets": [ + { + "key": "siem.eqlRule", + "doc_count": 381, + "rulesByEnabled": { + "doc_count_error_upper_bound": 0, + "sum_other_doc_count": 0, + "buckets": [ + { + "key": 1, + "key_as_string": "true", + "doc_count": 381 + } + ] + } + }, + { + "key": "siem.queryRule", + "doc_count": 325, + "rulesByEnabled": { + "doc_count_error_upper_bound": 0, + "sum_other_doc_count": 0, + "buckets": [ + { + "key": 1, + "key_as_string": "true", + "doc_count": 325 + } + ] + } + }, + { + "key": "siem.mlRule", + "doc_count": 47, + "rulesByEnabled": { + "doc_count_error_upper_bound": 0, + "sum_other_doc_count": 0, + "buckets": [ + { + "key": 1, + "key_as_string": "true", + "doc_count": 47 + } + ] + } + }, + { + "key": "siem.thresholdRule", + "doc_count": 18, + "rulesByEnabled": { + "doc_count_error_upper_bound": 0, + "sum_other_doc_count": 0, + "buckets": [ + { + "key": 1, + "key_as_string": "true", + "doc_count": 18 + } + ] + } + }, + { + "key": "siem.newTermsRule", + "doc_count": 4, + "rulesByEnabled": { + "doc_count_error_upper_bound": 0, + "sum_other_doc_count": 0, + "buckets": [ + { + "key": 1, + "key_as_string": "true", + "doc_count": 4 + } + ] + } + }, + { + "key": "siem.indicatorRule", + "doc_count": 2, + "rulesByEnabled": { + "doc_count_error_upper_bound": 0, + "sum_other_doc_count": 0, + "buckets": [ + { + "key": 1, + "key_as_string": "true", + "doc_count": 2 + } + ] + } + } + ] + }, + "rulesByOrigin": { + "doc_count_error_upper_bound": 0, + "sum_other_doc_count": 0, + "buckets": [ + { + "key": "true", + "doc_count": 776, + "rulesByEnabled": { + "doc_count_error_upper_bound": 0, + "sum_other_doc_count": 0, + "buckets": [ + { + "key": 1, + "key_as_string": "true", + "doc_count": 776 + } + ] + } + }, + { + "key": "false", + "doc_count": 1, + "rulesByEnabled": { + "doc_count_error_upper_bound": 0, + "sum_other_doc_count": 0, + "buckets": [ + { + "key": 1, + "key_as_string": "true", + "doc_count": 1 + } + ] + } + } + ] + }, + "rulesByEnabled": { + "doc_count_error_upper_bound": 0, + "sum_other_doc_count": 0, + "buckets": [ + { + "key": 1, + "key_as_string": "true", + "doc_count": 777 + } + ] + } + } + } + }, + "eventLog": { + "request": { + "aggs": { + "totalExecutions": { + "cardinality": { + "field": "kibana.alert.rule.execution.uuid" + } + }, + "executeEvents": { + "filter": { + "term": { + "event.action": "execute" + } + }, + "aggs": { + "executionDurationMs": { + "percentiles": { + "field": "kibana.alert.rule.execution.metrics.total_run_duration_ms", + "missing": 0, + "percents": [1, 5, 25, 50, 75, 95, 99] + } + }, + "scheduleDelayNs": { + "percentiles": { + "field": "kibana.task.schedule_delay", + "missing": 0, + "percents": [1, 5, 25, 50, 75, 95, 99] + } + } + } + }, + "statusChangeEvents": { + "filter": { + "bool": { + "filter": [ + { + "term": { + "event.action": "status-change" + } + } + ], + "must_not": [ + { + "terms": { + "kibana.alert.rule.execution.status": ["running", "going to run"] + } + } + ] + } + }, + "aggs": { + "executionsByStatus": { + "terms": { + "field": "kibana.alert.rule.execution.status" + } + } + } + }, + "executionMetricsEvents": { + "filter": { + "term": { + "event.action": "execution-metrics" + } + }, + "aggs": { + "gaps": { + "filter": { + "exists": { + "field": "kibana.alert.rule.execution.metrics.execution_gap_duration_s" + } + }, + "aggs": { + "totalGapDurationS": { + "sum": { + "field": "kibana.alert.rule.execution.metrics.execution_gap_duration_s" + } + } + } + }, + "searchDurationMs": { + "percentiles": { + "field": "kibana.alert.rule.execution.metrics.total_search_duration_ms", + "missing": 0, + "percents": [1, 5, 25, 50, 75, 95, 99] + } + }, + "indexingDurationMs": { + "percentiles": { + "field": "kibana.alert.rule.execution.metrics.total_indexing_duration_ms", + "missing": 0, + "percents": [1, 5, 25, 50, 75, 95, 99] + } + } + } + }, + "messageContainingEvents": { + "filter": { + "terms": { + "event.action": ["status-change", "message"] + } + }, + "aggs": { + "messagesByLogLevel": { + "terms": { + "field": "log.level" + } + }, + "errors": { + "filter": { + "term": { + "log.level": "error" + } + }, + "aggs": { + "topErrors": { + "categorize_text": { + "field": "message", + "size": 5, + "similarity_threshold": 99 + } + } + } + }, + "warnings": { + "filter": { + "term": { + "log.level": "warn" + } + }, + "aggs": { + "topWarnings": { + "categorize_text": { + "field": "message", + "size": 5, + "similarity_threshold": 99 + } + } + } + } + } + }, + "statsHistory": { + "date_histogram": { + "field": "@timestamp", + "calendar_interval": "hour" + }, + "aggs": { + "totalExecutions": { + "cardinality": { + "field": "kibana.alert.rule.execution.uuid" + } + }, + "executeEvents": { + "filter": { + "term": { + "event.action": "execute" + } + }, + "aggs": { + "executionDurationMs": { + "percentiles": { + "field": "kibana.alert.rule.execution.metrics.total_run_duration_ms", + "missing": 0, + "percents": [1, 5, 25, 50, 75, 95, 99] + } + }, + "scheduleDelayNs": { + "percentiles": { + "field": "kibana.task.schedule_delay", + "missing": 0, + "percents": [1, 5, 25, 50, 75, 95, 99] + } + } + } + }, + "statusChangeEvents": { + "filter": { + "bool": { + "filter": [ + { + "term": { + "event.action": "status-change" + } + } + ], + "must_not": [ + { + "terms": { + "kibana.alert.rule.execution.status": ["running", "going to run"] + } + } + ] + } + }, + "aggs": { + "executionsByStatus": { + "terms": { + "field": "kibana.alert.rule.execution.status" + } + } + } + }, + "executionMetricsEvents": { + "filter": { + "term": { + "event.action": "execution-metrics" + } + }, + "aggs": { + "gaps": { + "filter": { + "exists": { + "field": "kibana.alert.rule.execution.metrics.execution_gap_duration_s" + } + }, + "aggs": { + "totalGapDurationS": { + "sum": { + "field": "kibana.alert.rule.execution.metrics.execution_gap_duration_s" + } + } + } + }, + "searchDurationMs": { + "percentiles": { + "field": "kibana.alert.rule.execution.metrics.total_search_duration_ms", + "missing": 0, + "percents": [1, 5, 25, 50, 75, 95, 99] + } + }, + "indexingDurationMs": { + "percentiles": { + "field": "kibana.alert.rule.execution.metrics.total_indexing_duration_ms", + "missing": 0, + "percents": [1, 5, 25, 50, 75, 95, 99] + } + } + } + }, + "messageContainingEvents": { + "filter": { + "terms": { + "event.action": ["status-change", "message"] + } + }, + "aggs": { + "messagesByLogLevel": { + "terms": { + "field": "log.level" + } + } + } + } + } + } + } + }, + "response": { + "aggregations": { + "statsHistory": { + "buckets": [ + { + "key_as_string": "2023-05-26T15:00:00.000Z", + "key": 1685113200000, + "doc_count": 11388, + "statusChangeEvents": { + "doc_count": 2751, + "executionsByStatus": { + "doc_count_error_upper_bound": 0, + "sum_other_doc_count": 0, + "buckets": [ + { + "key": "failed", + "doc_count": 1336 + }, + { + "key": "partial failure", + "doc_count": 849 + }, + { + "key": "succeeded", + "doc_count": 566 + } + ] + } + }, + "totalExecutions": { + "value": 2245 + }, + "messageContainingEvents": { + "doc_count": 4996, + "messagesByLogLevel": { + "doc_count_error_upper_bound": 0, + "sum_other_doc_count": 0, + "buckets": [ + { + "key": "info", + "doc_count": 2811 + }, + { + "key": "error", + "doc_count": 1336 + }, + { + "key": "warn", + "doc_count": 849 + } + ] + } + }, + "executeEvents": { + "doc_count": 2245, + "scheduleDelayNs": { + "values": { + "1.0": 256000000, + "5.0": 3086972222.222222, + "25.0": 3133000000, + "50.0": 6126000000, + "75.0": 59484250000, + "95.0": 179817250000, + "99.0": 202613000000 + } + }, + "executionDurationMs": { + "values": { + "1.0": 280.6, + "5.0": 327.7, + "25.0": 371.5208333333333, + "50.0": 415.6190476190476, + "75.0": 505.575, + "95.0": 740.4375, + "99.0": 1446.1500000000005 + } + } + }, + "executionMetricsEvents": { + "doc_count": 1902, + "searchDurationMs": { + "values": { + "1.0": 0, + "5.0": 0, + "25.0": 0, + "50.0": 0, + "75.0": 8, + "95.0": 25, + "99.0": 46 + } + }, + "gaps": { + "doc_count": 777, + "totalGapDurationS": { + "value": 514415894 + } + }, + "indexingDurationMs": { + "values": { + "1.0": 0, + "5.0": 0, + "25.0": 0, + "50.0": 0, + "75.0": 0, + "95.0": 0, + "99.0": 0 + } + } + } + }, + { + "key_as_string": "2023-05-26T16:00:00.000Z", + "key": 1685116800000, + "doc_count": 28325, + "statusChangeEvents": { + "doc_count": 6126, + "executionsByStatus": { + "doc_count_error_upper_bound": 0, + "sum_other_doc_count": 0, + "buckets": [ + { + "key": "succeeded", + "doc_count": 2390 + }, + { + "key": "partial failure", + "doc_count": 2305 + }, + { + "key": "failed", + "doc_count": 1431 + } + ] + } + }, + "totalExecutions": { + "value": 6170 + }, + "messageContainingEvents": { + "doc_count": 12252, + "messagesByLogLevel": { + "doc_count_error_upper_bound": 0, + "sum_other_doc_count": 0, + "buckets": [ + { + "key": "info", + "doc_count": 8516 + }, + { + "key": "warn", + "doc_count": 2305 + }, + { + "key": "error", + "doc_count": 1431 + } + ] + } + }, + "executeEvents": { + "doc_count": 6126, + "scheduleDelayNs": { + "values": { + "1.0": 193000000, + "5.0": 3017785185.1851854, + "25.0": 3086000000, + "50.0": 3105877192.982456, + "75.0": 3134645161.290323, + "95.0": 6081772222.222222, + "99.0": 6122000000 + } + }, + "executionDurationMs": { + "values": { + "1.0": 275.17333333333335, + "5.0": 324.8014285714285, + "25.0": 377.0752688172043, + "50.0": 431, + "75.0": 532.3870967741935, + "95.0": 720.6761904761904, + "99.0": 922.6799999999985 + } + } + }, + "executionMetricsEvents": { + "doc_count": 3821, + "searchDurationMs": { + "values": { + "1.0": 0, + "5.0": 0, + "25.0": 0, + "50.0": 9.8, + "75.0": 18, + "95.0": 40.17499999999999, + "99.0": 124 + } + }, + "gaps": { + "doc_count": 0, + "totalGapDurationS": { + "value": 0 + } + }, + "indexingDurationMs": { + "values": { + "1.0": 0, + "5.0": 0, + "25.0": 0, + "50.0": 0, + "75.0": 0, + "95.0": 0, + "99.0": 0 + } + } + } + } + ] + }, + "statusChangeEvents": { + "doc_count": 8877, + "executionsByStatus": { + "doc_count_error_upper_bound": 0, + "sum_other_doc_count": 0, + "buckets": [ + { + "key": "partial failure", + "doc_count": 3154 + }, + { + "key": "succeeded", + "doc_count": 2956 + }, + { + "key": "failed", + "doc_count": 2767 + } + ] + } + }, + "totalExecutions": { + "value": 8455 + }, + "messageContainingEvents": { + "doc_count": 17248, + "messagesByLogLevel": { + "doc_count_error_upper_bound": 0, + "sum_other_doc_count": 0, + "buckets": [ + { + "key": "info", + "doc_count": 11327 + }, + { + "key": "warn", + "doc_count": 3154 + }, + { + "key": "error", + "doc_count": 2767 + } + ] + }, + "warnings": { + "doc_count": 3154, + "topWarnings": { + "buckets": [ + { + "doc_count": 3154, + "key": "This rule is attempting to query data from Elasticsearch indices listed in the Index pattern section of the rule definition however no index matching was found This warning will continue to appear until matching index is created or this rule is disabled", + "regex": ".*?This.+?rule.+?is.+?attempting.+?to.+?query.+?data.+?from.+?Elasticsearch.+?indices.+?listed.+?in.+?the.+?Index.+?pattern.+?section.+?of.+?the.+?rule.+?definition.+?however.+?no.+?index.+?matching.+?was.+?found.+?This.+?warning.+?will.+?continue.+?to.+?appear.+?until.+?matching.+?index.+?is.+?created.+?or.+?this.+?rule.+?is.+?disabled.*?", + "max_matching_length": 342 + } + ] + } + }, + "errors": { + "doc_count": 2767, + "topErrors": { + "buckets": [ + { + "doc_count": 1802, + "key": "An error occurred during rule execution message verification_exception", + "regex": ".*?An.+?error.+?occurred.+?during.+?rule.+?execution.+?message.+?verification_exception.*?", + "max_matching_length": 2064 + }, + { + "doc_count": 777, + "key": "were not queried between this rule execution and the last execution so signals may have been missed Consider increasing your look behind time or adding more Kibana instances", + "regex": ".*?were.+?not.+?queried.+?between.+?this.+?rule.+?execution.+?and.+?the.+?last.+?execution.+?so.+?signals.+?may.+?have.+?been.+?missed.+?Consider.+?increasing.+?your.+?look.+?behind.+?time.+?or.+?adding.+?more.+?Kibana.+?instances.*?", + "max_matching_length": 216 + }, + { + "doc_count": 4, + "key": "An error occurred during rule execution message rare_error_code missing", + "regex": ".*?An.+?error.+?occurred.+?during.+?rule.+?execution.+?message.+?rare_error_code.+?missing.*?", + "max_matching_length": 82 + }, + { + "doc_count": 4, + "key": "An error occurred during rule execution message v3_windows_anomalous_path_activity missing", + "regex": ".*?An.+?error.+?occurred.+?during.+?rule.+?execution.+?message.+?v3_windows_anomalous_path_activity.+?missing.*?", + "max_matching_length": 103 + }, + { + "doc_count": 4, + "key": "An error occurred during rule execution message v3_windows_rare_user_type10_remote_login missing", + "regex": ".*?An.+?error.+?occurred.+?during.+?rule.+?execution.+?message.+?v3_windows_rare_user_type10_remote_login.+?missing.*?", + "max_matching_length": 110 + } + ] + } + } + }, + "executeEvents": { + "doc_count": 8371, + "scheduleDelayNs": { + "values": { + "1.0": 206000000, + "5.0": 3027000000, + "25.0": 3092000000, + "50.0": 3116000000, + "75.0": 3278666666.6666665, + "95.0": 99656950000, + "99.0": 186632790000 + } + }, + "executionDurationMs": { + "values": { + "1.0": 275.5325, + "5.0": 326.07857142857137, + "25.0": 375.68969144460027, + "50.0": 427, + "75.0": 526.2948717948718, + "95.0": 727.2480952380952, + "99.0": 1009.5299999999934 + } + } + }, + "executionMetricsEvents": { + "doc_count": 5723, + "searchDurationMs": { + "values": { + "1.0": 0, + "5.0": 0, + "25.0": 0, + "50.0": 4, + "75.0": 16, + "95.0": 34.43846153846145, + "99.0": 116.51333333333302 + } + }, + "gaps": { + "doc_count": 777, + "totalGapDurationS": { + "value": 514415894 + } + }, + "indexingDurationMs": { + "values": { + "1.0": 0, + "5.0": 0, + "25.0": 0, + "50.0": 0, + "75.0": 0, + "95.0": 0, + "99.0": 0 + } + } + } + } + } + } + } + } +} +``` diff --git a/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/detection_engine_health/get_cluster_health_schemas.ts b/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/detection_engine_health/get_cluster_health_schemas.ts new file mode 100644 index 000000000000000..a4b28a2d1bc31c2 --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/detection_engine_health/get_cluster_health_schemas.ts @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import * as t from 'io-ts'; +import type { IsoDateString } from '@kbn/securitysolution-io-ts-types'; +import type { HealthInterval } from '../../model/detection_engine_health/health_interval'; +import { HealthIntervalParameters } from '../../model/detection_engine_health/health_interval'; +import type { HealthTimings } from '../../model/detection_engine_health/health_metadata'; +import type { + ClusterHealthParameters, + ClusterHealthSnapshot, +} from '../../model/detection_engine_health/cluster_health'; + +/** + * Schema for the request body of the endpoint. + */ +export type GetClusterHealthRequestBody = t.TypeOf; +export const GetClusterHealthRequestBody = t.exact( + t.partial({ + interval: HealthIntervalParameters, + debug: t.boolean, + }) +); + +/** + * Validated and normalized request parameters of the endpoint. + */ +export interface GetClusterHealthRequest { + /** + * Time period over which health stats are requested. + */ + interval: HealthInterval; + + /** + * If true, the endpoint will return various debug information, such as + * aggregations sent to Elasticsearch and response received from Elasticsearch. + */ + debug: boolean; + + /** + * Timestamp at which the route handler started executing. + */ + requestReceivedAt: IsoDateString; +} + +/** + * Response body of the endpoint. + */ +export interface GetClusterHealthResponse { + // TODO: https://github.com/elastic/kibana/issues/125642 Implement the endpoint and remove the `message` property + message: 'Not implemented'; + + /** + * Request processing times and durations. + */ + timings: HealthTimings; + + /** + * Parameters of the health stats calculation. + */ + parameters: ClusterHealthParameters; + + /** + * Result of the health stats calculation. + */ + health: ClusterHealthSnapshot; +} diff --git a/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/detection_engine_health/get_rule_health_schemas.ts b/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/detection_engine_health/get_rule_health_schemas.ts new file mode 100644 index 000000000000000..8f810549f54ee78 --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/detection_engine_health/get_rule_health_schemas.ts @@ -0,0 +1,80 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import * as t from 'io-ts'; +import type { IsoDateString } from '@kbn/securitysolution-io-ts-types'; +import { NonEmptyString } from '@kbn/securitysolution-io-ts-types'; + +import type { HealthInterval } from '../../model/detection_engine_health/health_interval'; +import { HealthIntervalParameters } from '../../model/detection_engine_health/health_interval'; +import type { HealthTimings } from '../../model/detection_engine_health/health_metadata'; +import type { + RuleHealthParameters, + RuleHealthSnapshot, +} from '../../model/detection_engine_health/rule_health'; + +/** + * Schema for the request body of the endpoint. + */ +export type GetRuleHealthRequestBody = t.TypeOf; +export const GetRuleHealthRequestBody = t.exact( + t.intersection([ + t.type({ + rule_id: NonEmptyString, + }), + t.partial({ + interval: HealthIntervalParameters, + debug: t.boolean, + }), + ]) +); + +/** + * Validated and normalized request parameters of the endpoint. + */ +export interface GetRuleHealthRequest { + /** + * Saved object ID of the rule to calculate health stats for. + */ + ruleId: string; + + /** + * Time period over which health stats are requested. + */ + interval: HealthInterval; + + /** + * If true, the endpoint will return various debug information, such as + * aggregations sent to Elasticsearch and response received from Elasticsearch. + */ + debug: boolean; + + /** + * Timestamp at which the route handler started executing. + */ + requestReceivedAt: IsoDateString; +} + +/** + * Response body of the endpoint. + */ +export interface GetRuleHealthResponse { + /** + * Request processing times and durations. + */ + timings: HealthTimings; + + /** + * Parameters of the health stats calculation. + */ + parameters: RuleHealthParameters; + + /** + * Result of the health stats calculation. + */ + health: RuleHealthSnapshot; +} diff --git a/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/detection_engine_health/get_space_health_schemas.ts b/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/detection_engine_health/get_space_health_schemas.ts new file mode 100644 index 000000000000000..4ad9a95a3f938b8 --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/detection_engine_health/get_space_health_schemas.ts @@ -0,0 +1,68 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import * as t from 'io-ts'; +import type { IsoDateString } from '@kbn/securitysolution-io-ts-types'; +import type { HealthInterval } from '../../model/detection_engine_health/health_interval'; +import { HealthIntervalParameters } from '../../model/detection_engine_health/health_interval'; +import type { HealthTimings } from '../../model/detection_engine_health/health_metadata'; +import type { + SpaceHealthParameters, + SpaceHealthSnapshot, +} from '../../model/detection_engine_health/space_health'; + +/** + * Schema for the request body of the endpoint. + */ +export type GetSpaceHealthRequestBody = t.TypeOf; +export const GetSpaceHealthRequestBody = t.exact( + t.partial({ + interval: HealthIntervalParameters, + debug: t.boolean, + }) +); + +/** + * Validated and normalized request parameters of the endpoint. + */ +export interface GetSpaceHealthRequest { + /** + * Time period over which health stats are requested. + */ + interval: HealthInterval; + + /** + * If true, the endpoint will return various debug information, such as + * aggregations sent to Elasticsearch and response received from Elasticsearch. + */ + debug: boolean; + + /** + * Timestamp at which the route handler started executing. + */ + requestReceivedAt: IsoDateString; +} + +/** + * Response body of the endpoint. + */ +export interface GetSpaceHealthResponse { + /** + * Request processing times and durations. + */ + timings: HealthTimings; + + /** + * Parameters of the health stats calculation. + */ + parameters: SpaceHealthParameters; + + /** + * Result of the health stats calculation. + */ + health: SpaceHealthSnapshot; +} diff --git a/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/get_rule_execution_events/response_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/get_rule_execution_events/response_schema.ts deleted file mode 100644 index 8637b3b0411a278..000000000000000 --- a/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/get_rule_execution_events/response_schema.ts +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import * as t from 'io-ts'; -import { PaginationResult } from '../../../schemas/common'; -import { RuleExecutionEvent } from '../../model/execution_event'; - -/** - * Response body of the API route. - */ -export type GetRuleExecutionEventsResponse = t.TypeOf; -export const GetRuleExecutionEventsResponse = t.exact( - t.type({ - events: t.array(RuleExecutionEvent), - pagination: PaginationResult, - }) -); diff --git a/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/get_rule_execution_results/response_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/get_rule_execution_results/response_schema.ts deleted file mode 100644 index 7610a21d1818185..000000000000000 --- a/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/get_rule_execution_results/response_schema.ts +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import * as t from 'io-ts'; -import { RuleExecutionResult } from '../../model/execution_result'; - -/** - * Response body of the API route. - */ -export type GetRuleExecutionResultsResponse = t.TypeOf; -export const GetRuleExecutionResultsResponse = t.exact( - t.type({ - events: t.array(RuleExecutionResult), - total: t.number, - }) -); diff --git a/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/get_rule_execution_events/response_schema.mock.ts b/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/rule_execution_logs/get_rule_execution_events_schemas.mock.ts similarity index 87% rename from x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/get_rule_execution_events/response_schema.mock.ts rename to x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/rule_execution_logs/get_rule_execution_events_schemas.mock.ts index c4c501f2aeac938..4d647b5f100947c 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/get_rule_execution_events/response_schema.mock.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/rule_execution_logs/get_rule_execution_events_schemas.mock.ts @@ -6,7 +6,7 @@ */ import { ruleExecutionEventMock } from '../../model/execution_event.mock'; -import type { GetRuleExecutionEventsResponse } from './response_schema'; +import type { GetRuleExecutionEventsResponse } from './get_rule_execution_events_schemas'; const getSomeResponse = (): GetRuleExecutionEventsResponse => { const events = ruleExecutionEventMock.getSomeEvents(); diff --git a/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/get_rule_execution_events/request_schema.test.ts b/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/rule_execution_logs/get_rule_execution_events_schemas.test.ts similarity index 99% rename from x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/get_rule_execution_events/request_schema.test.ts rename to x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/rule_execution_logs/get_rule_execution_events_schemas.test.ts index c6e02b6f815e361..89b7b992fbbadb3 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/get_rule_execution_events/request_schema.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/rule_execution_logs/get_rule_execution_events_schemas.test.ts @@ -12,7 +12,7 @@ import { foldLeftRight, getPaths } from '@kbn/securitysolution-io-ts-utils'; import { GetRuleExecutionEventsRequestParams, GetRuleExecutionEventsRequestQuery, -} from './request_schema'; +} from './get_rule_execution_events_schemas'; describe('Request schema of Get rule execution events', () => { describe('GetRuleExecutionEventsRequestParams', () => { diff --git a/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/get_rule_execution_events/request_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/rule_execution_logs/get_rule_execution_events_schemas.ts similarity index 73% rename from x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/get_rule_execution_events/request_schema.ts rename to x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/rule_execution_logs/get_rule_execution_events_schemas.ts index 59e17a9d6f60426..8c3c76e2286d95b 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/get_rule_execution_events/request_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/rule_execution_logs/get_rule_execution_events_schemas.ts @@ -10,8 +10,8 @@ import * as t from 'io-ts'; import { DefaultPerPage, DefaultPage } from '@kbn/securitysolution-io-ts-alerting-types'; import { defaultCsvArray, NonEmptyString } from '@kbn/securitysolution-io-ts-types'; -import { DefaultSortOrderDesc } from '../../../schemas/common'; -import { TRuleExecutionEventType } from '../../model/execution_event'; +import { DefaultSortOrderDesc, PaginationResult } from '../../../schemas/common'; +import { RuleExecutionEvent, TRuleExecutionEventType } from '../../model/execution_event'; import { TLogLevel } from '../../model/log_level'; /** @@ -41,3 +41,14 @@ export const GetRuleExecutionEventsRequestQuery = t.exact( per_page: DefaultPerPage, // defaults to 20 }) ); + +/** + * Response body of the API route. + */ +export type GetRuleExecutionEventsResponse = t.TypeOf; +export const GetRuleExecutionEventsResponse = t.exact( + t.type({ + events: t.array(RuleExecutionEvent), + pagination: PaginationResult, + }) +); diff --git a/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/get_rule_execution_results/response_schema.mock.ts b/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/rule_execution_logs/get_rule_execution_results_schemas.mock.ts similarity index 86% rename from x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/get_rule_execution_results/response_schema.mock.ts rename to x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/rule_execution_logs/get_rule_execution_results_schemas.mock.ts index fb6be3eeb62cc7c..b8e323e6fb1f362 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/get_rule_execution_results/response_schema.mock.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/rule_execution_logs/get_rule_execution_results_schemas.mock.ts @@ -6,7 +6,7 @@ */ import { ruleExecutionResultMock } from '../../model/execution_result.mock'; -import type { GetRuleExecutionResultsResponse } from './response_schema'; +import type { GetRuleExecutionResultsResponse } from './get_rule_execution_results_schemas'; const getSomeResponse = (): GetRuleExecutionResultsResponse => { const results = ruleExecutionResultMock.getSomeResults(); diff --git a/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/get_rule_execution_results/request_schema.test.ts b/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/rule_execution_logs/get_rule_execution_results_schemas.test.ts similarity index 98% rename from x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/get_rule_execution_results/request_schema.test.ts rename to x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/rule_execution_logs/get_rule_execution_results_schemas.test.ts index 8757084a2ec980d..375047b935d2475 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/get_rule_execution_results/request_schema.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/rule_execution_logs/get_rule_execution_results_schemas.test.ts @@ -10,7 +10,10 @@ import { left } from 'fp-ts/lib/Either'; import { foldLeftRight, getPaths } from '@kbn/securitysolution-io-ts-utils'; import { RULE_EXECUTION_STATUSES } from '../../model/execution_status'; -import { DefaultSortField, DefaultRuleExecutionStatusCsvArray } from './request_schema'; +import { + DefaultSortField, + DefaultRuleExecutionStatusCsvArray, +} from './get_rule_execution_results_schemas'; describe('Request schema of Get rule execution results', () => { describe('DefaultRuleExecutionStatusCsvArray', () => { diff --git a/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/get_rule_execution_results/request_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/rule_execution_logs/get_rule_execution_results_schemas.ts similarity index 86% rename from x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/get_rule_execution_results/request_schema.ts rename to x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/rule_execution_logs/get_rule_execution_results_schemas.ts index 33458ab0a875aca..e2cec9b7a3b2984 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/get_rule_execution_results/request_schema.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/rule_execution_logs/get_rule_execution_results_schemas.ts @@ -17,7 +17,7 @@ import { } from '@kbn/securitysolution-io-ts-types'; import { DefaultSortOrderDesc } from '../../../schemas/common'; -import { SortFieldOfRuleExecutionResult } from '../../model/execution_result'; +import { RuleExecutionResult, SortFieldOfRuleExecutionResult } from '../../model/execution_result'; import { TRuleExecutionStatus } from '../../model/execution_status'; /** @@ -70,3 +70,14 @@ export const GetRuleExecutionResultsRequestQuery = t.exact( per_page: DefaultPerPage, // defaults to 20 }) ); + +/** + * Response body of the API route. + */ +export type GetRuleExecutionResultsResponse = t.TypeOf; +export const GetRuleExecutionResultsResponse = t.exact( + t.type({ + events: t.array(RuleExecutionResult), + total: t.number, + }) +); diff --git a/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/urls.ts b/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/urls.ts index 595ca6c01d83a10..8e5a25227ece1ef 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/urls.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/urls.ts @@ -7,11 +7,43 @@ import { INTERNAL_DETECTION_ENGINE_URL as INTERNAL_URL } from '../../../constants'; +// ------------------------------------------------------------------------------------------------- +// Detection Engine health API + +/** + * Get health overview of the whole cluster. Scope: all detection rules in all Kibana spaces. + * See the corresponding route handler for more details. + */ +export const GET_CLUSTER_HEALTH_URL = `${INTERNAL_URL}/health/_cluster` as const; + +/** + * Get health overview of the current Kibana space. Scope: all detection rules in the space. + * See the corresponding route handler for more details. + */ +export const GET_SPACE_HEALTH_URL = `${INTERNAL_URL}/health/_space` as const; + +/** + * Get health overview of a rule. Scope: a given detection rule in the current Kibana space. + * See the corresponding route handler for more details. + */ +export const GET_RULE_HEALTH_URL = `${INTERNAL_URL}/health/_rule` as const; + +// ------------------------------------------------------------------------------------------------- +// Rule execution logs API + +/** + * Get plain individual rule execution events, such as status changes, execution metrics, + * log messages, etc. + */ export const GET_RULE_EXECUTION_EVENTS_URL = `${INTERNAL_URL}/rules/{ruleId}/execution/events` as const; export const getRuleExecutionEventsUrl = (ruleId: string) => `${INTERNAL_URL}/rules/${ruleId}/execution/events` as const; +/** + * Get aggregated rule execution results. Each result object is built on top of all individual + * events logged during the corresponding rule execution. + */ export const GET_RULE_EXECUTION_RESULTS_URL = `${INTERNAL_URL}/rules/{ruleId}/execution/results` as const; export const getRuleExecutionResultsUrl = (ruleId: string) => diff --git a/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/index.ts b/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/index.ts index b7881e2d2f52470..c4e6ccc4f1b3a7b 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/index.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/index.ts @@ -5,12 +5,19 @@ * 2.0. */ -export * from './api/get_rule_execution_events/request_schema'; -export * from './api/get_rule_execution_events/response_schema'; -export * from './api/get_rule_execution_results/request_schema'; -export * from './api/get_rule_execution_results/response_schema'; +export * from './api/detection_engine_health/get_cluster_health_schemas'; +export * from './api/detection_engine_health/get_rule_health_schemas'; +export * from './api/detection_engine_health/get_space_health_schemas'; +export * from './api/rule_execution_logs/get_rule_execution_events_schemas'; +export * from './api/rule_execution_logs/get_rule_execution_results_schemas'; export * from './api/urls'; +export * from './model/detection_engine_health/cluster_health'; +export * from './model/detection_engine_health/health_interval'; +export * from './model/detection_engine_health/health_metadata'; +export * from './model/detection_engine_health/health_stats'; +export * from './model/detection_engine_health/rule_health'; +export * from './model/detection_engine_health/space_health'; export * from './model/execution_event'; export * from './model/execution_metrics'; export * from './model/execution_result'; diff --git a/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/mocks.ts b/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/mocks.ts index a552a0fc047f60e..1a93749849cdfba 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/mocks.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/mocks.ts @@ -5,9 +5,12 @@ * 2.0. */ -export * from './api/get_rule_execution_events/response_schema.mock'; -export * from './api/get_rule_execution_results/response_schema.mock'; +export * from './api/rule_execution_logs/get_rule_execution_events_schemas.mock'; +export * from './api/rule_execution_logs/get_rule_execution_results_schemas.mock'; +export * from './model/detection_engine_health/cluster_health.mock'; +export * from './model/detection_engine_health/rule_health.mock'; +export * from './model/detection_engine_health/space_health.mock'; export * from './model/execution_event.mock'; export * from './model/execution_result.mock'; export * from './model/execution_summary.mock'; diff --git a/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/model/detection_engine_health/cluster_health.mock.ts b/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/model/detection_engine_health/cluster_health.mock.ts new file mode 100644 index 000000000000000..6bd740a87cf5453 --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/model/detection_engine_health/cluster_health.mock.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ClusterHealthSnapshot } from './cluster_health'; +import { healthStatsMock } from './health_stats.mock'; + +const getEmptyClusterHealthSnapshot = (): ClusterHealthSnapshot => { + return { + stats_at_the_moment: healthStatsMock.getEmptyRuleStats(), + stats_over_interval: { + message: 'Not implemented', + }, + history_over_interval: { + buckets: [ + { + timestamp: '2023-05-15T16:12:14.967Z', + stats: { + message: 'Not implemented', + }, + }, + ], + }, + }; +}; + +export const clusterHealthSnapshotMock = { + getEmptyClusterHealthSnapshot, +}; diff --git a/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/model/detection_engine_health/cluster_health.ts b/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/model/detection_engine_health/cluster_health.ts new file mode 100644 index 000000000000000..441eef935ade550 --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/model/detection_engine_health/cluster_health.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { HealthParameters, HealthSnapshot } from './health_metadata'; +import type { RuleStats, StatsHistory } from './health_stats'; + +/** + * Health calculation parameters for the whole cluster. + */ +export type ClusterHealthParameters = HealthParameters; + +/** + * Health calculation result for the whole cluster. + */ +export interface ClusterHealthSnapshot extends HealthSnapshot { + /** + * Health stats at the moment of the calculation request. + */ + stats_at_the_moment: ClusterHealthStatsAtTheMoment; + + /** + * Health stats calculated over the interval specified in the health parameters. + */ + stats_over_interval: ClusterHealthStatsOverInterval; + + /** + * History of change of the same health stats during the interval. + */ + history_over_interval: StatsHistory; +} + +/** + * Health stats at the moment of the calculation request. + */ +export type ClusterHealthStatsAtTheMoment = RuleStats; + +/** + * Health stats calculated over a given interval. + */ +export interface ClusterHealthStatsOverInterval { + // TODO: https://github.com/elastic/kibana/issues/125642 Implement and delete this `message` + message: 'Not implemented'; +} diff --git a/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/model/detection_engine_health/health_interval.ts b/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/model/detection_engine_health/health_interval.ts new file mode 100644 index 000000000000000..3a073b546d6eba9 --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/model/detection_engine_health/health_interval.ts @@ -0,0 +1,138 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import * as t from 'io-ts'; +import { IsoDateString } from '@kbn/securitysolution-io-ts-types'; + +/** + * Type of the health interval. You can specify: + * - a relative interval, e.g. "last_hour" = [now-1h; now] where "now" is when health request is made + * - a custom interval with "from" and "to" timestamps + */ +export enum HealthIntervalType { + 'last_hour' = 'last_hour', + 'last_day' = 'last_day', + 'last_week' = 'last_week', + 'last_month' = 'last_month', + 'last_year' = 'last_year', + 'custom_range' = 'custom_range', +} + +/** + * Granularity defines how the whole health interval will be split into smaller sub-intervals. + * Health stats will be calculated for the whole interval + for each sub-interval. + * Example: if the interval is "last_day" and the granularity is "hour", stats will be calculated: + * - 1 time for the last 24 hours + * - 24 times for each hour in that interval + */ +export enum HealthIntervalGranularity { + 'minute' = 'minute', + 'hour' = 'hour', + 'day' = 'day', + 'week' = 'week', + 'month' = 'month', +} + +/** + * Time period over which we calculate health stats. + * This is a "raw" schema for the interval parameters that users can pass to the API. + */ +export type HealthIntervalParameters = t.TypeOf; +export const HealthIntervalParameters = t.union([ + t.exact( + t.type({ + type: t.literal(HealthIntervalType.last_hour), + granularity: t.literal(HealthIntervalGranularity.minute), + }) + ), + t.exact( + t.type({ + type: t.literal(HealthIntervalType.last_day), + granularity: t.union([ + t.literal(HealthIntervalGranularity.minute), + t.literal(HealthIntervalGranularity.hour), + ]), + }) + ), + t.exact( + t.type({ + type: t.literal(HealthIntervalType.last_week), + granularity: t.union([ + t.literal(HealthIntervalGranularity.hour), + t.literal(HealthIntervalGranularity.day), + ]), + }) + ), + t.exact( + t.type({ + type: t.literal(HealthIntervalType.last_month), + granularity: t.union([ + t.literal(HealthIntervalGranularity.day), + t.literal(HealthIntervalGranularity.week), + ]), + }) + ), + t.exact( + t.type({ + type: t.literal(HealthIntervalType.last_year), + granularity: t.union([ + t.literal(HealthIntervalGranularity.week), + t.literal(HealthIntervalGranularity.month), + ]), + }) + ), + t.exact( + t.type({ + type: t.literal(HealthIntervalType.custom_range), + granularity: t.union([ + t.literal(HealthIntervalGranularity.minute), + t.literal(HealthIntervalGranularity.hour), + t.literal(HealthIntervalGranularity.day), + t.literal(HealthIntervalGranularity.week), + t.literal(HealthIntervalGranularity.month), + ]), + from: IsoDateString, + to: IsoDateString, + }) + ), +]); + +/** + * Time period over which we calculate health stats. + * This interface represents a fully validated and normalized interval object. + */ +export interface HealthInterval { + /** + * Type of the interval. Defined by the user. + * @example 'last_week' + */ + type: HealthIntervalType; + + /** + * Granularity of the interval. Defined by the user. + * @example 'day' + */ + granularity: HealthIntervalGranularity; + + /** + * Start timestamp of the interval. Calculated by the app. + * @example '2023-05-19T14:25:19.092Z' + */ + from: IsoDateString; + + /** + * End timestamp of the interval. Calculated by the app. + * @example '2023-05-26T14:25:19.092Z' + */ + to: IsoDateString; + + /** + * Duration of the interval in the ISO format. Calculated by the app. + * @example 'PT168H' + */ + duration: string; +} diff --git a/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/model/detection_engine_health/health_metadata.ts b/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/model/detection_engine_health/health_metadata.ts new file mode 100644 index 000000000000000..37d7242542386dd --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/model/detection_engine_health/health_metadata.ts @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IsoDateString } from '@kbn/securitysolution-io-ts-types'; +import type { HealthInterval } from './health_interval'; + +/** + * Health request processing times and durations. + * This metadata is included in the health API responses. + */ +export interface HealthTimings { + /** + * Timestamp at which health calculation request was received. + */ + requested_at: IsoDateString; + + /** + * Timestamp at which health stats were calculated and returned. + */ + processed_at: IsoDateString; + + /** + * How much time it took to calculate health stats, in milliseconds. + */ + processing_time_ms: number; +} + +/** + * Base parameters of all the health API endpoints. + * This metadata is included in the health API responses. + */ +export interface HealthParameters { + /** + * Time period over which we calculate health stats. + */ + interval: HealthInterval; +} + +/** + * Base properties of a health snapshot (health calculation result at a given moment). + */ +export interface HealthSnapshot { + /** + * Optional debug information, such as requests and aggregations sent to Elasticsearch + * and responses received from Elasticsearch. + */ + debug?: Record; +} diff --git a/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/model/detection_engine_health/health_stats.mock.ts b/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/model/detection_engine_health/health_stats.mock.ts new file mode 100644 index 000000000000000..545a1b0ef0440b2 --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/model/detection_engine_health/health_stats.mock.ts @@ -0,0 +1,87 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + AggregatedMetric, + RuleExecutionStats, + RuleStats, + TotalEnabledDisabled, +} from './health_stats'; + +const getEmptyRuleStats = (): RuleStats => { + return { + number_of_rules: { + all: getZeroTotalEnabledDisabled(), + by_origin: { + prebuilt: getZeroTotalEnabledDisabled(), + custom: getZeroTotalEnabledDisabled(), + }, + by_type: {}, + by_outcome: {}, + }, + }; +}; + +const getZeroTotalEnabledDisabled = (): TotalEnabledDisabled => { + return { + total: 0, + enabled: 0, + disabled: 0, + }; +}; + +const getEmptyRuleExecutionStats = (): RuleExecutionStats => { + return { + number_of_executions: { + total: 0, + by_outcome: { + succeeded: 0, + warning: 0, + failed: 0, + }, + }, + number_of_logged_messages: { + total: 0, + by_level: { + error: 0, + warn: 0, + info: 0, + debug: 0, + trace: 0, + }, + }, + number_of_detected_gaps: { + total: 0, + total_duration_s: 0, + }, + schedule_delay_ms: getZeroAggregatedMetric(), + execution_duration_ms: getZeroAggregatedMetric(), + search_duration_ms: getZeroAggregatedMetric(), + indexing_duration_ms: getZeroAggregatedMetric(), + top_errors: [], + top_warnings: [], + }; +}; + +const getZeroAggregatedMetric = (): AggregatedMetric => { + return { + percentiles: { + '1.0': 0, + '5.0': 0, + '25.0': 0, + '50.0': 0, + '75.0': 0, + '95.0': 0, + '99.0': 0, + }, + }; +}; + +export const healthStatsMock = { + getEmptyRuleStats, + getEmptyRuleExecutionStats, +}; diff --git a/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/model/detection_engine_health/health_stats.ts b/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/model/detection_engine_health/health_stats.ts new file mode 100644 index 000000000000000..e43a5bfbd3073fa --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/model/detection_engine_health/health_stats.ts @@ -0,0 +1,269 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IsoDateString } from '@kbn/securitysolution-io-ts-types'; +import type { RuleLastRunOutcomes } from '@kbn/alerting-plugin/common'; +import type { LogLevel } from '../log_level'; + +// ------------------------------------------------------------------------------------------------- +// Stats history (date histogram) + +/** + * History of change of a set of stats over a time interval. The interval is split into discreet buckets, + * each bucket is a smaller sub-interval with stats calculated over this sub-interval. + * + * This model corresponds to the `date_histogram` aggregation of Elasticsearch: + * https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-bucket-datehistogram-aggregation.html + */ +export interface StatsHistory { + buckets: Array>; +} + +/** + * Sub-interval with stats calculated over it. + */ +export interface StatsBucket { + /** + * Start timestamp of the sub-interval. + */ + timestamp: IsoDateString; + + /** + * Set of stats. + */ + stats: TStats; +} + +// ------------------------------------------------------------------------------------------------- +// Rule stats + +// TODO: https://github.com/elastic/kibana/issues/125642 Add more stats, such as: +// - number of Kibana instances +// - number of Kibana spaces +// - number of rules with exceptions +// - number of rules with notification actions (total, normal, legacy) +// - number of rules with response actions +// - top X last failed status messages + rule ids for each status +// - top X last partial failure status messages + rule ids for each status +// - top X slowest rules by any metrics (last total execution time, search time, indexing time, etc) +// - top X rules with the largest schedule delay (drift) + +/** + * "Static" stats calculated for a set of rules, such as number of enabled and disabled rules, etc. + */ +export interface RuleStats { + /** + * Various counts of different rules. + */ + number_of_rules: NumberOfRules; +} + +/** + * Various counts of different rules. + */ +export interface NumberOfRules { + /** + * Total number of all rules, and how many of them are enabled and disabled. + */ + all: TotalEnabledDisabled; + + /** + * Number of prebuilt and custom rules, and how many of them are enabled and disabled. + */ + by_origin: Record<'prebuilt' | 'custom', TotalEnabledDisabled>; + + /** + * Number of rules of each type, and how many of them are enabled and disabled. + */ + by_type: Record; + + /** + * Number of rules by last execution outcome, and how many of them are enabled and disabled. + */ + by_outcome: Record; +} + +/** + * Number of rules in a given set, and how many of them are enabled and disabled. + */ +export interface TotalEnabledDisabled { + /** + * Total number of rules in a set. + */ + total: number; + + /** + * Number of enabled rules in a set. + */ + enabled: number; + + /** + * Number of disabled rules in a set. + */ + disabled: number; +} + +// ------------------------------------------------------------------------------------------------- +// Rule execution stats + +// TODO: https://github.com/elastic/kibana/issues/125642 Add more stats, such as: +// - number of detected alerts (source event "hits") +// - number of created alerts (those we wrote to the .alerts-* index) +// - number of times rule hit cirquit breaker, number of not created alerts because of that +// - number of triggered actions +// - top gaps + +/** + * "Dynamic" rule execution stats. Can be calculated either for a set of rules or for a single rule. + */ +export interface RuleExecutionStats { + /** + * Number of rule executions. + */ + number_of_executions: NumberOfExecutions; + + /** + * Number of events containing some message that were written to the Event Log. + */ + number_of_logged_messages: NumberOfLoggedMessages; + + /** + * Stats for detected gaps in rule execution. + */ + number_of_detected_gaps: NumberOfDetectedGaps; + + /** + * Aggregated schedule delay of a rule, in milliseconds. + * Also called "drift" in the Task Manager health API. + * This metric shows if rules start executing on time according to their schedule + * (in that case, it should be ideally zero, but in practice will be 3-5 seconds), + * or their start time gets delayed (when the cluster is overloaded it could be + * minutes or even hours). + */ + schedule_delay_ms: AggregatedMetric; + + /** + * Aggregated total execution duration of a rule, in milliseconds. + */ + execution_duration_ms: AggregatedMetric; + + /** + * Aggregated total search duration of a rule, in milliseconds. + * This metric shows how much time a rule spends for querying source indices. + */ + search_duration_ms: AggregatedMetric; + + /** + * Aggregated total indexing duration of a rule, in milliseconds. + * This metric shows how much time a rule spends for writing generated alerts. + */ + indexing_duration_ms: AggregatedMetric; + + /** + * N most frequent error messages logged by rule(s) to Event Log. + */ + top_errors?: TopMessages; + + /** + * N most frequent warning messages logged by rule(s) to Event Log. + */ + top_warnings?: TopMessages; +} + +/** + * Number of rule executions. + */ +export interface NumberOfExecutions { + /** + * Total number of rule executions. + */ + total: number; + + /** + * Number of executions by each possible execution outcome. + */ + by_outcome: Record; +} + +/** + * Number of events containing some message that were written to the Event Log. + */ +export interface NumberOfLoggedMessages { + /** + * Total number of message-containing events. + */ + total: number; + + /** + * Number of message-containing events by each log level. + */ + by_level: Record; +} + +/** + * Stats for detected gaps in rule execution. + */ +export interface NumberOfDetectedGaps { + /** + * Total number of detected gaps. + */ + total: number; + + /** + * Sum of durations of all the detected gaps, in seconds. + */ + total_duration_s: number; +} + +/** + * When a rule runs, we calculate a bunch of rule execution metrics for a given rule run. + * Later, we can aggregate each metric in different ways: + * - for a single rule, aggregate over a time interval + * - for multiple rules, aggregate over a time interval + * - for multiple rules, aggregate over the rules at a given moment (e.g. now) + * + * For example, if the metric is "total rule execution duration", we could: + * - calculate average execution duration of a single rule over last week + * - calculate average execution duration of all rules in a space over last week + * - calculate average last execution duration of all rules in a space at the moment + * + * Instead of calculating only averages, we calculate a set of percentiles that can give + * a better picture of the metric's distribution. + */ +export interface AggregatedMetric { + percentiles: Percentiles; +} + +/** + * Distribution of values of an aggregated metric represented by a set of discreet percentiles. + * @example + * { + * '1.0': 27, + * '5.0': 150, + * '25.0': 240, + * '50.0': 420, + * '75.0': 700, + * '95.0': 2500, + * '99.0': 7800, + * } + */ +export type Percentiles = Record; + +/** + * Most frequent messages logged by rule(s) to Event Log. + */ +export type TopMessages = Array<{ + /** + * Number of occurencies of a message. + */ + count: number; + + /** + * The message itself. + */ + message: string; +}>; diff --git a/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/model/detection_engine_health/rule_health.mock.ts b/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/model/detection_engine_health/rule_health.mock.ts new file mode 100644 index 000000000000000..2ae5b8a7a82054f --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/model/detection_engine_health/rule_health.mock.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getRulesSchemaMock } from '../../../rule_schema/mocks'; +import { healthStatsMock } from './health_stats.mock'; +import type { RuleHealthSnapshot } from './rule_health'; + +const getEmptyRuleHealthSnapshot = (): RuleHealthSnapshot => { + return { + stats_at_the_moment: { + rule: getRulesSchemaMock(), + }, + stats_over_interval: healthStatsMock.getEmptyRuleExecutionStats(), + history_over_interval: { + buckets: [ + { + timestamp: '2023-05-15T16:12:14.967Z', + stats: healthStatsMock.getEmptyRuleExecutionStats(), + }, + ], + }, + }; +}; + +export const ruleHealthSnapshotMock = { + getEmptyRuleHealthSnapshot, +}; diff --git a/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/model/detection_engine_health/rule_health.ts b/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/model/detection_engine_health/rule_health.ts new file mode 100644 index 000000000000000..b0b3577a46967be --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/model/detection_engine_health/rule_health.ts @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { RuleResponse } from '../../../rule_schema/model/rule_schemas'; +import type { HealthParameters, HealthSnapshot } from './health_metadata'; +import type { RuleExecutionStats, StatsHistory } from './health_stats'; + +/** + * Health calculation parameters for a given rule. + */ +export interface RuleHealthParameters extends HealthParameters { + /** + * Saved object ID of the rule. + */ + rule_id: string; +} + +/** + * Health calculation result for a given rule. + */ +export interface RuleHealthSnapshot extends HealthSnapshot { + /** + * Health stats at the moment of the calculation request. + */ + stats_at_the_moment: RuleHealthStatsAtTheMoment; + + /** + * Health stats calculated over the interval specified in the health parameters. + */ + stats_over_interval: RuleHealthStatsOverInterval; + + /** + * History of change of the same health stats during the interval. + */ + history_over_interval: StatsHistory; +} + +/** + * Health stats at the moment of the calculation request. + */ +export interface RuleHealthStatsAtTheMoment { + /** + * Rule object including its current execution summary. + */ + rule: RuleResponse; +} + +/** + * Health stats calculated over a given interval. + */ +export type RuleHealthStatsOverInterval = RuleExecutionStats; diff --git a/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/model/detection_engine_health/space_health.mock.ts b/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/model/detection_engine_health/space_health.mock.ts new file mode 100644 index 000000000000000..60e1514cee59e99 --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/model/detection_engine_health/space_health.mock.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { healthStatsMock } from './health_stats.mock'; +import type { SpaceHealthSnapshot } from './space_health'; + +const getEmptySpaceHealthSnapshot = (): SpaceHealthSnapshot => { + return { + stats_at_the_moment: healthStatsMock.getEmptyRuleStats(), + stats_over_interval: healthStatsMock.getEmptyRuleExecutionStats(), + history_over_interval: { + buckets: [ + { + timestamp: '2023-05-15T16:12:14.967Z', + stats: healthStatsMock.getEmptyRuleExecutionStats(), + }, + ], + }, + }; +}; + +export const spaceHealthSnapshotMock = { + getEmptySpaceHealthSnapshot, +}; diff --git a/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/model/detection_engine_health/space_health.ts b/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/model/detection_engine_health/space_health.ts new file mode 100644 index 000000000000000..35648a9257570d8 --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/model/detection_engine_health/space_health.ts @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { HealthParameters, HealthSnapshot } from './health_metadata'; +import type { RuleExecutionStats, RuleStats, StatsHistory } from './health_stats'; + +/** + * Health calculation parameters for the current Kibana space. + */ +export type SpaceHealthParameters = HealthParameters; + +/** + * Health calculation result for the current Kibana space. + */ +export interface SpaceHealthSnapshot extends HealthSnapshot { + /** + * Health stats at the moment of the calculation request. + */ + stats_at_the_moment: SpaceHealthStatsAtTheMoment; + + /** + * Health stats calculated over the interval specified in the health parameters. + */ + stats_over_interval: SpaceHealthStatsOverInterval; + + /** + * History of change of the same health stats during the interval. + */ + history_over_interval: StatsHistory; +} + +/** + * Health stats at the moment of the calculation request. + */ +export type SpaceHealthStatsAtTheMoment = RuleStats; + +/** + * Health stats calculated over a given interval. + */ +export type SpaceHealthStatsOverInterval = RuleExecutionStats; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_context.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_context.ts index bfda85063f2b93d..0d7969e8e88d0c4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_context.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_context.ts @@ -23,7 +23,7 @@ import { ruleRegistryMocks } from '@kbn/rule-registry-plugin/server/mocks'; import { siemMock } from '../../../../mocks'; import { createMockConfig } from '../../../../config.mock'; -import { ruleExecutionLogMock } from '../../rule_monitoring/mocks'; +import { detectionEngineHealthClientMock, ruleExecutionLogMock } from '../../rule_monitoring/mocks'; import { requestMock } from './request'; import { internalFrameworkRequest } from '../../../framework'; @@ -58,6 +58,8 @@ export const createMockClients = () => { config: createMockConfig(), appClient: siemMock.createClient(), + + detectionEngineHealthClient: detectionEngineHealthClientMock.create(), ruleExecutionLog: ruleExecutionLogMock.forRoutes.create(), }; }; @@ -129,6 +131,7 @@ const createSecuritySolutionRequestContextMock = ( }), getSpaceId: jest.fn(() => 'default'), getRuleDataService: jest.fn(() => clients.ruleDataService), + getDetectionEngineHealthClient: jest.fn(() => clients.detectionEngineHealthClient), getRuleExecutionLog: jest.fn(() => clients.ruleExecutionLog), getExceptionListClient: jest.fn(() => clients.lists.exceptionListClient), getInternalFleetServices: jest.fn(() => { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/read_rule/route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/read_rule/route.ts index 64e1daec632ba65..da7e4be16e3c4f3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/read_rule/route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_management/api/rules/read_rule/route.ts @@ -44,6 +44,7 @@ export const readRuleRoute = (router: SecuritySolutionPluginRouter, logger: Logg try { const rulesClient = (await context.alerting).getRulesClient(); + // TODO: https://github.com/elastic/kibana/issues/125642 Reuse fetchRuleById const rule = await readRules({ id, rulesClient, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/detection_engine_health/get_cluster_health/get_cluster_health_request.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/detection_engine_health/get_cluster_health/get_cluster_health_request.ts new file mode 100644 index 000000000000000..162febd917ac76c --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/detection_engine_health/get_cluster_health/get_cluster_health_request.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import moment from 'moment'; +import type { + GetClusterHealthRequest, + GetClusterHealthRequestBody, +} from '../../../../../../../common/detection_engine/rule_monitoring'; +import { validateHealthInterval } from '../health_interval'; + +export const validateGetClusterHealthRequest = ( + body: GetClusterHealthRequestBody +): GetClusterHealthRequest => { + const now = moment(); + const interval = validateHealthInterval(body.interval, now); + + return { + interval, + debug: body.debug ?? false, + requestReceivedAt: now.utc().toISOString(), + }; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/detection_engine_health/get_cluster_health/get_cluster_health_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/detection_engine_health/get_cluster_health/get_cluster_health_route.ts new file mode 100644 index 000000000000000..1e3bda7bab1cfd8 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/detection_engine_health/get_cluster_health/get_cluster_health_route.ts @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { transformError } from '@kbn/securitysolution-es-utils'; +import { buildRouteValidation } from '../../../../../../utils/build_validation/route_validation'; +import { buildSiemResponse } from '../../../../routes/utils'; +import type { SecuritySolutionPluginRouter } from '../../../../../../types'; + +import type { GetClusterHealthResponse } from '../../../../../../../common/detection_engine/rule_monitoring'; +import { + GET_CLUSTER_HEALTH_URL, + GetClusterHealthRequestBody, +} from '../../../../../../../common/detection_engine/rule_monitoring'; +import { calculateHealthTimings } from '../health_timings'; +import { validateGetClusterHealthRequest } from './get_cluster_health_request'; + +/** + * Get health overview of the whole cluster. Scope: all detection rules in all Kibana spaces. + * Returns: + * - health stats at the moment of the API call + * - health stats over a specified period of time ("health interval") + * - health stats history within the same interval in the form of a histogram + * (the same stats are calculated over each of the discreet sub-intervals of the whole interval) + */ +export const getClusterHealthRoute = (router: SecuritySolutionPluginRouter) => { + router.post( + { + path: GET_CLUSTER_HEALTH_URL, + validate: { + body: buildRouteValidation(GetClusterHealthRequestBody), + }, + options: { + tags: ['access:securitySolution'], + }, + }, + async (context, request, response) => { + const siemResponse = buildSiemResponse(response); + + try { + const params = validateGetClusterHealthRequest(request.body); + + const ctx = await context.resolve(['securitySolution']); + const healthClient = ctx.securitySolution.getDetectionEngineHealthClient(); + + const clusterHealthParameters = { interval: params.interval }; + const clusterHealth = await healthClient.calculateClusterHealth(clusterHealthParameters); + + const responseBody: GetClusterHealthResponse = { + // TODO: https://github.com/elastic/kibana/issues/125642 Implement the endpoint and remove the `message` property + message: 'Not implemented', + timings: calculateHealthTimings(params.requestReceivedAt), + parameters: clusterHealthParameters, + health: { + ...clusterHealth, + debug: params.debug ? clusterHealth.debug : undefined, + }, + }; + + return response.ok({ body: responseBody }); + } catch (err) { + const error = transformError(err); + return siemResponse.error({ + body: error.message, + statusCode: error.statusCode, + }); + } + } + ); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/detection_engine_health/get_rule_health/get_rule_health_request.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/detection_engine_health/get_rule_health/get_rule_health_request.ts new file mode 100644 index 000000000000000..8a5f0a1dd7f782e --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/detection_engine_health/get_rule_health/get_rule_health_request.ts @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import moment from 'moment'; +import type { + GetRuleHealthRequest, + GetRuleHealthRequestBody, +} from '../../../../../../../common/detection_engine/rule_monitoring'; +import { validateHealthInterval } from '../health_interval'; + +export const validateGetRuleHealthRequest = ( + body: GetRuleHealthRequestBody +): GetRuleHealthRequest => { + const now = moment(); + const interval = validateHealthInterval(body.interval, now); + + return { + ruleId: body.rule_id, + interval, + debug: body.debug ?? false, + requestReceivedAt: now.utc().toISOString(), + }; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/detection_engine_health/get_rule_health/get_rule_health_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/detection_engine_health/get_rule_health/get_rule_health_route.ts new file mode 100644 index 000000000000000..71ed3f63c716225 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/detection_engine_health/get_rule_health/get_rule_health_route.ts @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { transformError } from '@kbn/securitysolution-es-utils'; + +import type { GetRuleHealthResponse } from '../../../../../../../common/detection_engine/rule_monitoring'; +import { + GetRuleHealthRequestBody, + GET_RULE_HEALTH_URL, +} from '../../../../../../../common/detection_engine/rule_monitoring'; + +import type { SecuritySolutionPluginRouter } from '../../../../../../types'; +import { buildRouteValidation } from '../../../../../../utils/build_validation/route_validation'; +import { buildSiemResponse } from '../../../../routes/utils'; +import { calculateHealthTimings } from '../health_timings'; +import { validateGetRuleHealthRequest } from './get_rule_health_request'; + +/** + * Get health overview of a rule. Scope: a given detection rule in the current Kibana space. + * Returns: + * - health stats at the moment of the API call (rule and its execution summary) + * - health stats over a specified period of time ("health interval") + * - health stats history within the same interval in the form of a histogram + * (the same stats are calculated over each of the discreet sub-intervals of the whole interval) + */ +export const getRuleHealthRoute = (router: SecuritySolutionPluginRouter) => { + router.post( + { + path: GET_RULE_HEALTH_URL, + validate: { + body: buildRouteValidation(GetRuleHealthRequestBody), + }, + options: { + tags: ['access:securitySolution'], + }, + }, + async (context, request, response) => { + const siemResponse = buildSiemResponse(response); + + try { + const params = validateGetRuleHealthRequest(request.body); + + const ctx = await context.resolve(['securitySolution']); + const healthClient = ctx.securitySolution.getDetectionEngineHealthClient(); + + const ruleHealthParameters = { interval: params.interval, rule_id: params.ruleId }; + const ruleHealth = await healthClient.calculateRuleHealth(ruleHealthParameters); + + const responseBody: GetRuleHealthResponse = { + timings: calculateHealthTimings(params.requestReceivedAt), + parameters: ruleHealthParameters, + health: { + ...ruleHealth, + debug: params.debug ? ruleHealth.debug : undefined, + }, + }; + + return response.ok({ body: responseBody }); + } catch (err) { + const error = transformError(err); + return siemResponse.error({ + body: error.message, + statusCode: error.statusCode, + }); + } + } + ); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/detection_engine_health/get_space_health/get_space_health_request.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/detection_engine_health/get_space_health/get_space_health_request.ts new file mode 100644 index 000000000000000..d74069c07c8a168 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/detection_engine_health/get_space_health/get_space_health_request.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import moment from 'moment'; +import type { + GetSpaceHealthRequest, + GetSpaceHealthRequestBody, +} from '../../../../../../../common/detection_engine/rule_monitoring'; +import { validateHealthInterval } from '../health_interval'; + +export const validateGetSpaceHealthRequest = ( + body: GetSpaceHealthRequestBody +): GetSpaceHealthRequest => { + const now = moment(); + const interval = validateHealthInterval(body.interval, now); + + return { + interval, + debug: body.debug ?? false, + requestReceivedAt: now.utc().toISOString(), + }; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/detection_engine_health/get_space_health/get_space_health_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/detection_engine_health/get_space_health/get_space_health_route.ts new file mode 100644 index 000000000000000..c70e197633adccc --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/detection_engine_health/get_space_health/get_space_health_route.ts @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { transformError } from '@kbn/securitysolution-es-utils'; +import { buildRouteValidation } from '../../../../../../utils/build_validation/route_validation'; +import { buildSiemResponse } from '../../../../routes/utils'; +import type { SecuritySolutionPluginRouter } from '../../../../../../types'; + +import type { GetSpaceHealthResponse } from '../../../../../../../common/detection_engine/rule_monitoring'; +import { + GET_SPACE_HEALTH_URL, + GetSpaceHealthRequestBody, +} from '../../../../../../../common/detection_engine/rule_monitoring'; +import { calculateHealthTimings } from '../health_timings'; +import { validateGetSpaceHealthRequest } from './get_space_health_request'; + +/** + * Get health overview of the current Kibana space. Scope: all detection rules in the space. + * Returns: + * - health stats at the moment of the API call + * - health stats over a specified period of time ("health interval") + * - health stats history within the same interval in the form of a histogram + * (the same stats are calculated over each of the discreet sub-intervals of the whole interval) + */ +export const getSpaceHealthRoute = (router: SecuritySolutionPluginRouter) => { + router.post( + { + path: GET_SPACE_HEALTH_URL, + validate: { + body: buildRouteValidation(GetSpaceHealthRequestBody), + }, + options: { + tags: ['access:securitySolution'], + }, + }, + async (context, request, response) => { + const siemResponse = buildSiemResponse(response); + + try { + const params = validateGetSpaceHealthRequest(request.body); + + const ctx = await context.resolve(['securitySolution']); + const healthClient = ctx.securitySolution.getDetectionEngineHealthClient(); + + const spaceHealthParameters = { interval: params.interval }; + const spaceHealth = await healthClient.calculateSpaceHealth(spaceHealthParameters); + + const responseBody: GetSpaceHealthResponse = { + timings: calculateHealthTimings(params.requestReceivedAt), + parameters: spaceHealthParameters, + health: { + ...spaceHealth, + debug: params.debug ? spaceHealth.debug : undefined, + }, + }; + + return response.ok({ body: responseBody }); + } catch (err) { + const error = transformError(err); + return siemResponse.error({ + body: error.message, + statusCode: error.statusCode, + }); + } + } + ); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/detection_engine_health/health_interval.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/detection_engine_health/health_interval.ts new file mode 100644 index 000000000000000..b6eb276c4f84964 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/detection_engine_health/health_interval.ts @@ -0,0 +1,90 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import moment from 'moment'; + +import type { + HealthInterval, + HealthIntervalParameters, +} from '../../../../../../common/detection_engine/rule_monitoring'; +import { + HealthIntervalGranularity, + HealthIntervalType, +} from '../../../../../../common/detection_engine/rule_monitoring'; +import { assertUnreachable } from '../../../../../../common/utility_types'; + +const DEFAULT_INTERVAL_PARAMETERS: HealthIntervalParameters = { + type: HealthIntervalType.last_day, + granularity: HealthIntervalGranularity.hour, +}; + +export const validateHealthInterval = ( + params: HealthIntervalParameters | undefined, + now: moment.Moment +): HealthInterval => { + const parameters = params ?? DEFAULT_INTERVAL_PARAMETERS; + + const from = getFrom(parameters, now); + const to = getTo(parameters, now); + const duration = moment.duration(to.diff(from)); + + // TODO: https://github.com/elastic/kibana/issues/125642 Validate that: + // - to > from + // - granularity is not too big, e.g. < duration (could be invalid when custom_range) + // - granularity is not too small (could be invalid when custom_range) + + return { + type: parameters.type, + granularity: parameters.granularity, + from: from.utc().toISOString(), + to: to.utc().toISOString(), + duration: duration.toISOString(), + }; +}; + +const getFrom = (params: HealthIntervalParameters, now: moment.Moment): moment.Moment => { + const { type } = params; + + // NOTE: it's important to clone `now` with `moment(now)` because moment objects are mutable. + // If you call .subtract() or other methods on the original `now`, you will change it which + // might cause bugs depending on how you use it in your calculations later. + + if (type === HealthIntervalType.custom_range) { + return moment(params.from); + } + if (type === HealthIntervalType.last_hour) { + return moment(now).subtract(1, 'hour'); + } + if (type === HealthIntervalType.last_day) { + return moment(now).subtract(1, 'day'); + } + if (type === HealthIntervalType.last_week) { + return moment(now).subtract(1, 'week'); + } + if (type === HealthIntervalType.last_month) { + return moment(now).subtract(1, 'month'); + } + if (type === HealthIntervalType.last_year) { + return moment(now).subtract(1, 'year'); + } + + return assertUnreachable(type, 'Unhandled health interval type'); +}; + +const getTo = (params: HealthIntervalParameters, now: moment.Moment): moment.Moment => { + const { type } = params; + + if (type === HealthIntervalType.custom_range) { + return moment(params.to); + } + + // NOTE: it's important to clone `now` with `moment(now)` because moment objects are mutable. If you + // return the original now from this method and then call .subtract() or other methods on it, it will + // change the original now which might cause bugs depending on how you use it in your calculations later. + + return moment(now); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/detection_engine_health/health_timings.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/detection_engine_health/health_timings.ts new file mode 100644 index 000000000000000..edbfa1ce8f8345d --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/detection_engine_health/health_timings.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import moment from 'moment'; +import type { IsoDateString } from '@kbn/securitysolution-io-ts-types'; +import type { HealthTimings } from '../../../../../../common/detection_engine/rule_monitoring'; + +export const calculateHealthTimings = (requestReceivedAt: IsoDateString): HealthTimings => { + const requestedAt = moment(requestReceivedAt); + const processedAt = moment().utc(); + const processingTime = moment.duration(processedAt.diff(requestReceivedAt)); + + return { + requested_at: requestedAt.toISOString(), + processed_at: processedAt.toISOString(), + processing_time_ms: processingTime.asMilliseconds(), + }; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/register_routes.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/register_routes.ts index c63cf638e7df214..ba376d15a84029f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/register_routes.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/register_routes.ts @@ -6,10 +6,19 @@ */ import type { SecuritySolutionPluginRouter } from '../../../../types'; -import { getRuleExecutionEventsRoute } from './get_rule_execution_events/route'; -import { getRuleExecutionResultsRoute } from './get_rule_execution_results/route'; +import { getClusterHealthRoute } from './detection_engine_health/get_cluster_health/get_cluster_health_route'; +import { getRuleHealthRoute } from './detection_engine_health/get_rule_health/get_rule_health_route'; +import { getSpaceHealthRoute } from './detection_engine_health/get_space_health/get_space_health_route'; +import { getRuleExecutionEventsRoute } from './rule_execution_logs/get_rule_execution_events/get_rule_execution_events_route'; +import { getRuleExecutionResultsRoute } from './rule_execution_logs/get_rule_execution_results/get_rule_execution_results_route'; export const registerRuleMonitoringRoutes = (router: SecuritySolutionPluginRouter) => { + // Detection Engine health API + getClusterHealthRoute(router); + getSpaceHealthRoute(router); + getRuleHealthRoute(router); + + // Rule execution logs API getRuleExecutionEventsRoute(router); getRuleExecutionResultsRoute(router); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/get_rule_execution_events/route.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/rule_execution_logs/get_rule_execution_events/get_rule_execution_events_route.test.ts similarity index 90% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/get_rule_execution_events/route.test.ts rename to x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/rule_execution_logs/get_rule_execution_events/get_rule_execution_events_route.test.ts index 519be6d429e9d2a..1f5d01387fce90a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/get_rule_execution_events/route.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/rule_execution_logs/get_rule_execution_events/get_rule_execution_events_route.test.ts @@ -5,16 +5,16 @@ * 2.0. */ -import { serverMock, requestContextMock, requestMock } from '../../../routes/__mocks__'; +import { serverMock, requestContextMock, requestMock } from '../../../../routes/__mocks__'; import { GET_RULE_EXECUTION_EVENTS_URL, LogLevel, RuleExecutionEventType, -} from '../../../../../../common/detection_engine/rule_monitoring'; -import { getRuleExecutionEventsResponseMock } from '../../../../../../common/detection_engine/rule_monitoring/mocks'; -import type { GetExecutionEventsArgs } from '../../logic/rule_execution_log'; -import { getRuleExecutionEventsRoute } from './route'; +} from '../../../../../../../common/detection_engine/rule_monitoring'; +import { getRuleExecutionEventsResponseMock } from '../../../../../../../common/detection_engine/rule_monitoring/mocks'; +import type { GetExecutionEventsArgs } from '../../../logic/rule_execution_log'; +import { getRuleExecutionEventsRoute } from './get_rule_execution_events_route'; describe('getRuleExecutionEventsRoute', () => { let server: ReturnType; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/get_rule_execution_events/route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/rule_execution_logs/get_rule_execution_events/get_rule_execution_events_route.ts similarity index 87% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/get_rule_execution_events/route.ts rename to x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/rule_execution_logs/get_rule_execution_events/get_rule_execution_events_route.ts index 109c15409afae42..9c7fb1f130af59f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/get_rule_execution_events/route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/rule_execution_logs/get_rule_execution_events/get_rule_execution_events_route.ts @@ -6,16 +6,16 @@ */ import { transformError } from '@kbn/securitysolution-es-utils'; -import { buildRouteValidation } from '../../../../../utils/build_validation/route_validation'; -import { buildSiemResponse } from '../../../routes/utils'; -import type { SecuritySolutionPluginRouter } from '../../../../../types'; +import { buildRouteValidation } from '../../../../../../utils/build_validation/route_validation'; +import { buildSiemResponse } from '../../../../routes/utils'; +import type { SecuritySolutionPluginRouter } from '../../../../../../types'; -import type { GetRuleExecutionEventsResponse } from '../../../../../../common/detection_engine/rule_monitoring'; +import type { GetRuleExecutionEventsResponse } from '../../../../../../../common/detection_engine/rule_monitoring'; import { GET_RULE_EXECUTION_EVENTS_URL, GetRuleExecutionEventsRequestParams, GetRuleExecutionEventsRequestQuery, -} from '../../../../../../common/detection_engine/rule_monitoring'; +} from '../../../../../../../common/detection_engine/rule_monitoring'; /** * Returns execution events of a given rule (e.g. status changes) from Event Log. diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/get_rule_execution_results/route.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/rule_execution_logs/get_rule_execution_results/get_rule_execution_results_route.test.ts similarity index 91% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/get_rule_execution_results/route.test.ts rename to x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/rule_execution_logs/get_rule_execution_results/get_rule_execution_results_route.test.ts index e041670c0b631e6..5eec91d39e6df73 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/get_rule_execution_results/route.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/rule_execution_logs/get_rule_execution_results/get_rule_execution_results_route.test.ts @@ -5,11 +5,11 @@ * 2.0. */ -import { serverMock, requestContextMock, requestMock } from '../../../routes/__mocks__'; +import { serverMock, requestContextMock, requestMock } from '../../../../routes/__mocks__'; -import { GET_RULE_EXECUTION_RESULTS_URL } from '../../../../../../common/detection_engine/rule_monitoring'; -import { getRuleExecutionResultsResponseMock } from '../../../../../../common/detection_engine/rule_monitoring/mocks'; -import { getRuleExecutionResultsRoute } from './route'; +import { GET_RULE_EXECUTION_RESULTS_URL } from '../../../../../../../common/detection_engine/rule_monitoring'; +import { getRuleExecutionResultsResponseMock } from '../../../../../../../common/detection_engine/rule_monitoring/mocks'; +import { getRuleExecutionResultsRoute } from './get_rule_execution_results_route'; describe('getRuleExecutionResultsRoute', () => { let server: ReturnType; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/get_rule_execution_results/route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/rule_execution_logs/get_rule_execution_results/get_rule_execution_results_route.ts similarity index 88% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/get_rule_execution_results/route.ts rename to x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/rule_execution_logs/get_rule_execution_results/get_rule_execution_results_route.ts index ff1523502aaeac7..5eaec11a3a7096f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/get_rule_execution_results/route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/rule_execution_logs/get_rule_execution_results/get_rule_execution_results_route.ts @@ -6,16 +6,16 @@ */ import { transformError } from '@kbn/securitysolution-es-utils'; -import { buildRouteValidation } from '../../../../../utils/build_validation/route_validation'; -import { buildSiemResponse } from '../../../routes/utils'; -import type { SecuritySolutionPluginRouter } from '../../../../../types'; +import { buildRouteValidation } from '../../../../../../utils/build_validation/route_validation'; +import { buildSiemResponse } from '../../../../routes/utils'; +import type { SecuritySolutionPluginRouter } from '../../../../../../types'; -import type { GetRuleExecutionResultsResponse } from '../../../../../../common/detection_engine/rule_monitoring'; +import type { GetRuleExecutionResultsResponse } from '../../../../../../../common/detection_engine/rule_monitoring'; import { GET_RULE_EXECUTION_RESULTS_URL, GetRuleExecutionResultsRequestParams, GetRuleExecutionResultsRequestQuery, -} from '../../../../../../common/detection_engine/rule_monitoring'; +} from '../../../../../../../common/detection_engine/rule_monitoring'; /** * Returns execution results of a given rule (aggregated by execution UUID) from Event Log. diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/index.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/index.ts index ca1b22776c247b3..9031a5949394c2b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/index.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/index.ts @@ -6,4 +6,9 @@ */ export * from './api/register_routes'; +export { RULE_EXECUTION_LOG_PROVIDER } from './logic/event_log/event_log_constants'; +export * from './logic/detection_engine_health'; export * from './logic/rule_execution_log'; +export * from './logic/service_interface'; +export * from './logic/service'; +export { truncateList } from './logic/utils/normalization'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/detection_engine_health/__mocks__/index.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/detection_engine_health/__mocks__/index.ts new file mode 100644 index 000000000000000..eb4566518c93235 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/detection_engine_health/__mocks__/index.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + clusterHealthSnapshotMock, + ruleHealthSnapshotMock, + spaceHealthSnapshotMock, +} from '../../../../../../../common/detection_engine/rule_monitoring/mocks'; + +import type { IDetectionEngineHealthClient } from '../detection_engine_health_client_interface'; + +type CalculateRuleHealth = IDetectionEngineHealthClient['calculateRuleHealth']; +type CalculateSpaceHealth = IDetectionEngineHealthClient['calculateSpaceHealth']; +type CalculateClusterHealth = IDetectionEngineHealthClient['calculateClusterHealth']; + +export const detectionEngineHealthClientMock = { + create: (): jest.Mocked => ({ + calculateRuleHealth: jest + .fn, Parameters>() + .mockResolvedValue(ruleHealthSnapshotMock.getEmptyRuleHealthSnapshot()), + + calculateSpaceHealth: jest + .fn, Parameters>() + .mockResolvedValue(spaceHealthSnapshotMock.getEmptySpaceHealthSnapshot()), + + calculateClusterHealth: jest + .fn, Parameters>() + .mockResolvedValue(clusterHealthSnapshotMock.getEmptyClusterHealthSnapshot()), + }), +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/detection_engine_health/detection_engine_health_client.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/detection_engine_health/detection_engine_health_client.ts new file mode 100644 index 000000000000000..232731e762ea91e --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/detection_engine_health/detection_engine_health_client.ts @@ -0,0 +1,122 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/core/server'; +import { withSecuritySpan } from '../../../../../utils/with_security_span'; +import type { ExtMeta } from '../utils/console_logging'; + +import type { + ClusterHealthParameters, + ClusterHealthSnapshot, + RuleHealthParameters, + RuleHealthSnapshot, + SpaceHealthParameters, + SpaceHealthSnapshot, +} from '../../../../../../common/detection_engine/rule_monitoring'; + +import type { IEventLogHealthClient } from './event_log/event_log_health_client'; +import type { IRuleObjectsHealthClient } from './rule_objects/rule_objects_health_client'; +import type { IDetectionEngineHealthClient } from './detection_engine_health_client_interface'; + +export const createDetectionEngineHealthClient = ( + ruleObjectsHealthClient: IRuleObjectsHealthClient, + eventLogHealthClient: IEventLogHealthClient, + logger: Logger, + currentSpaceId: string +): IDetectionEngineHealthClient => { + return { + calculateRuleHealth: (args: RuleHealthParameters): Promise => { + return withSecuritySpan('IDetectionEngineHealthClient.calculateRuleHealth', async () => { + const ruleId = args.rule_id; + try { + // We call these two sequentially, because if the rule doesn't exist we need to throw 404 + // from ruleObjectsHealthClient before we calculate expensive stats in eventLogHealthClient. + const statsBasedOnRuleObjects = await ruleObjectsHealthClient.calculateRuleHealth(args); + const statsBasedOnEventLog = await eventLogHealthClient.calculateRuleHealth(args); + + return { + stats_at_the_moment: statsBasedOnRuleObjects.stats_at_the_moment, + stats_over_interval: statsBasedOnEventLog.stats_over_interval, + history_over_interval: statsBasedOnEventLog.history_over_interval, + debug: { + ...statsBasedOnRuleObjects.debug, + ...statsBasedOnEventLog.debug, + }, + }; + } catch (e) { + const logMessage = 'Error calculating rule health'; + const logReason = e instanceof Error ? e.message : String(e); + const logSuffix = `[rule id ${ruleId}]`; + const logMeta: ExtMeta = { + rule: { id: ruleId }, + }; + + logger.error(`${logMessage}: ${logReason} ${logSuffix}`, logMeta); + throw e; + } + }); + }, + + calculateSpaceHealth: (args: SpaceHealthParameters): Promise => { + return withSecuritySpan('IDetectionEngineHealthClient.calculateSpaceHealth', async () => { + try { + const [statsBasedOnRuleObjects, statsBasedOnEventLog] = await Promise.all([ + ruleObjectsHealthClient.calculateSpaceHealth(args), + eventLogHealthClient.calculateSpaceHealth(args), + ]); + + return { + stats_at_the_moment: statsBasedOnRuleObjects.stats_at_the_moment, + stats_over_interval: statsBasedOnEventLog.stats_over_interval, + history_over_interval: statsBasedOnEventLog.history_over_interval, + debug: { + ...statsBasedOnRuleObjects.debug, + ...statsBasedOnEventLog.debug, + }, + }; + } catch (e) { + const logMessage = 'Error calculating space health'; + const logReason = e instanceof Error ? e.message : String(e); + const logSuffix = `[space id ${currentSpaceId}]`; + const logMeta: ExtMeta = { + kibana: { spaceId: currentSpaceId }, + }; + + logger.error(`${logMessage}: ${logReason} ${logSuffix}`, logMeta); + throw e; + } + }); + }, + + calculateClusterHealth: (args: ClusterHealthParameters): Promise => { + return withSecuritySpan('IDetectionEngineHealthClient.calculateClusterHealth', async () => { + try { + const [statsBasedOnRuleObjects, statsBasedOnEventLog] = await Promise.all([ + ruleObjectsHealthClient.calculateClusterHealth(args), + eventLogHealthClient.calculateClusterHealth(args), + ]); + + return { + stats_at_the_moment: statsBasedOnRuleObjects.stats_at_the_moment, + stats_over_interval: statsBasedOnEventLog.stats_over_interval, + history_over_interval: statsBasedOnEventLog.history_over_interval, + debug: { + ...statsBasedOnRuleObjects.debug, + ...statsBasedOnEventLog.debug, + }, + }; + } catch (e) { + const logMessage = 'Error calculating cluster health'; + const logReason = e instanceof Error ? e.message : String(e); + + logger.error(`${logMessage}: ${logReason}`); + throw e; + } + }); + }, + }; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/detection_engine_health/detection_engine_health_client_interface.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/detection_engine_health/detection_engine_health_client_interface.ts new file mode 100644 index 000000000000000..e24a5b9fca2eb3f --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/detection_engine_health/detection_engine_health_client_interface.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + ClusterHealthParameters, + ClusterHealthSnapshot, + RuleHealthParameters, + RuleHealthSnapshot, + SpaceHealthParameters, + SpaceHealthSnapshot, +} from '../../../../../../common/detection_engine/rule_monitoring'; + +/** + * Calculates health of the Detection Engine overall and detection rules individually. + */ +export interface IDetectionEngineHealthClient { + /** + * Calculates health stats for a given rule. + */ + calculateRuleHealth(args: RuleHealthParameters): Promise; + + /** + * Calculates health stats for all rules in the current Kibana space. + */ + calculateSpaceHealth(args: SpaceHealthParameters): Promise; + + /** + * Calculates health stats for the whole cluster. + */ + calculateClusterHealth(args: ClusterHealthParameters): Promise; +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/detection_engine_health/event_log/aggregations/health_stats_for_rule.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/detection_engine_health/event_log/aggregations/health_stats_for_rule.ts new file mode 100644 index 000000000000000..b02d6f3aad981b1 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/detection_engine_health/event_log/aggregations/health_stats_for_rule.ts @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import type { AggregateEventsBySavedObjectResult } from '@kbn/event-log-plugin/server'; + +import type { + HealthIntervalGranularity, + RuleHealthSnapshot, + RuleHealthStatsOverInterval, + StatsHistory, +} from '../../../../../../../../common/detection_engine/rule_monitoring'; +import type { RawData } from '../../../utils/normalization'; + +import * as f from '../../../event_log/event_log_fields'; +import { + getRuleExecutionStatsAggregation, + normalizeRuleExecutionStatsAggregationResult, +} from './rule_execution_stats'; + +export const getRuleHealthAggregation = ( + granularity: HealthIntervalGranularity +): Record => { + // Let's say we want to calculate rule execution statistics over some date interval, where: + // - the whole interval is one week (7 days) + // - the interval's granularity is one day + // This means we will be calculating the same rule execution stats: + // - One time over the whole week. + // - Seven times over a day, per each day in the week. + return { + // And so this function creates several aggs that will be calculated for the whole interval. + ...getRuleExecutionStatsAggregation('whole-interval'), + // And this one creates a histogram, where for each bucket we will calculate the same aggs. + // The histogram's "calendar_interval" is equal to the granularity parameter. + ...getRuleExecutionStatsHistoryAggregation(granularity), + }; +}; + +const getRuleExecutionStatsHistoryAggregation = ( + granularity: HealthIntervalGranularity +): Record => { + return { + statsHistory: { + date_histogram: { + field: f.TIMESTAMP, + calendar_interval: granularity, + }, + aggs: getRuleExecutionStatsAggregation('histogram'), + }, + }; +}; + +export const normalizeRuleHealthAggregationResult = ( + result: AggregateEventsBySavedObjectResult, + requestAggs: Record +): Omit => { + const aggregations = result.aggregations ?? {}; + return { + stats_over_interval: normalizeRuleExecutionStatsAggregationResult( + aggregations, + 'whole-interval' + ), + history_over_interval: normalizeHistoryOverInterval(aggregations), + debug: { + eventLog: { + request: { aggs: requestAggs }, + response: { aggregations }, + }, + }, + }; +}; + +const normalizeHistoryOverInterval = ( + aggregations: Record +): StatsHistory => { + const statsHistory = aggregations.statsHistory || {}; + + return { + buckets: statsHistory.buckets.map((rawBucket: RawData) => { + const timestamp: string = String(rawBucket.key_as_string); + const stats = normalizeRuleExecutionStatsAggregationResult(rawBucket, 'histogram'); + return { timestamp, stats }; + }), + }; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/detection_engine_health/event_log/aggregations/rule_execution_stats.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/detection_engine_health/event_log/aggregations/rule_execution_stats.ts new file mode 100644 index 000000000000000..16e3ccf5bc25d3d --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/detection_engine_health/event_log/aggregations/rule_execution_stats.ts @@ -0,0 +1,290 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { mapValues } from 'lodash'; +import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; + +import type { + AggregatedMetric, + NumberOfDetectedGaps, + NumberOfExecutions, + NumberOfLoggedMessages, + RuleExecutionStats, + TopMessages, +} from '../../../../../../../../common/detection_engine/rule_monitoring'; +import { + RuleExecutionEventType, + RuleExecutionStatus, + LogLevel, +} from '../../../../../../../../common/detection_engine/rule_monitoring'; + +import { DEFAULT_PERCENTILES } from '../../../utils/es_aggregations'; +import type { RawData } from '../../../utils/normalization'; +import * as f from '../../../event_log/event_log_fields'; + +export type RuleExecutionStatsAggregationLevel = 'whole-interval' | 'histogram'; + +export const getRuleExecutionStatsAggregation = ( + aggregationContext: RuleExecutionStatsAggregationLevel +): Record => { + return { + totalExecutions: { + cardinality: { + field: f.RULE_EXECUTION_UUID, + }, + }, + executeEvents: { + filter: { + term: { [f.EVENT_ACTION]: 'execute' }, + }, + aggs: { + executionDurationMs: { + percentiles: { + field: f.RULE_EXECUTION_TOTAL_DURATION_MS, + missing: 0, + percents: DEFAULT_PERCENTILES, + }, + }, + scheduleDelayNs: { + percentiles: { + field: f.RULE_EXECUTION_SCHEDULE_DELAY_NS, + missing: 0, + percents: DEFAULT_PERCENTILES, + }, + }, + }, + }, + statusChangeEvents: { + filter: { + bool: { + filter: [ + { + term: { + [f.EVENT_ACTION]: RuleExecutionEventType['status-change'], + }, + }, + ], + must_not: [ + { + terms: { + [f.RULE_EXECUTION_STATUS]: [ + RuleExecutionStatus.running, + RuleExecutionStatus['going to run'], + ], + }, + }, + ], + }, + }, + aggs: { + executionsByStatus: { + terms: { + field: f.RULE_EXECUTION_STATUS, + }, + }, + }, + }, + executionMetricsEvents: { + filter: { + term: { [f.EVENT_ACTION]: RuleExecutionEventType['execution-metrics'] }, + }, + aggs: { + gaps: { + filter: { + exists: { + field: f.RULE_EXECUTION_GAP_DURATION_S, + }, + }, + aggs: { + totalGapDurationS: { + sum: { + field: f.RULE_EXECUTION_GAP_DURATION_S, + }, + }, + }, + }, + searchDurationMs: { + percentiles: { + field: f.RULE_EXECUTION_SEARCH_DURATION_MS, + missing: 0, + percents: DEFAULT_PERCENTILES, + }, + }, + indexingDurationMs: { + percentiles: { + field: f.RULE_EXECUTION_INDEXING_DURATION_MS, + missing: 0, + percents: DEFAULT_PERCENTILES, + }, + }, + }, + }, + messageContainingEvents: { + filter: { + terms: { + [f.EVENT_ACTION]: [ + RuleExecutionEventType['status-change'], + RuleExecutionEventType.message, + ], + }, + }, + aggs: { + messagesByLogLevel: { + terms: { + field: f.LOG_LEVEL, + }, + }, + ...(aggregationContext === 'whole-interval' + ? { + errors: { + filter: { + term: { [f.LOG_LEVEL]: LogLevel.error }, + }, + aggs: { + topErrors: { + categorize_text: { + field: 'message', + size: 5, + similarity_threshold: 99, + }, + }, + }, + }, + warnings: { + filter: { + term: { [f.LOG_LEVEL]: LogLevel.warn }, + }, + aggs: { + topWarnings: { + categorize_text: { + field: 'message', + size: 5, + similarity_threshold: 99, + }, + }, + }, + }, + } + : {}), + }, + }, + }; +}; + +export const normalizeRuleExecutionStatsAggregationResult = ( + aggregations: Record, + aggregationLevel: RuleExecutionStatsAggregationLevel +): RuleExecutionStats => { + const totalExecutions = aggregations.totalExecutions || {}; + const executeEvents = aggregations.executeEvents || {}; + const statusChangeEvents = aggregations.statusChangeEvents || {}; + const executionMetricsEvents = aggregations.executionMetricsEvents || {}; + const messageContainingEvents = aggregations.messageContainingEvents || {}; + + const executionDurationMs = executeEvents.executionDurationMs || {}; + const scheduleDelayNs = executeEvents.scheduleDelayNs || {}; + const executionsByStatus = statusChangeEvents.executionsByStatus || {}; + const gaps = executionMetricsEvents.gaps || {}; + const searchDurationMs = executionMetricsEvents.searchDurationMs || {}; + const indexingDurationMs = executionMetricsEvents.indexingDurationMs || {}; + + return { + number_of_executions: normalizeNumberOfExecutions(totalExecutions, executionsByStatus), + number_of_logged_messages: normalizeNumberOfLoggedMessages(messageContainingEvents), + number_of_detected_gaps: normalizeNumberOfDetectedGaps(gaps), + schedule_delay_ms: normalizeAggregatedMetric(scheduleDelayNs, (val) => val / 1_000_000), + execution_duration_ms: normalizeAggregatedMetric(executionDurationMs), + search_duration_ms: normalizeAggregatedMetric(searchDurationMs), + indexing_duration_ms: normalizeAggregatedMetric(indexingDurationMs), + top_errors: + aggregationLevel === 'whole-interval' + ? normalizeTopErrors(messageContainingEvents) + : undefined, + top_warnings: + aggregationLevel === 'whole-interval' + ? normalizeTopWarnings(messageContainingEvents) + : undefined, + }; +}; + +const normalizeNumberOfExecutions = ( + totalExecutions: RawData, + executionsByStatus: RawData +): NumberOfExecutions => { + const getStatusCount = (status: RuleExecutionStatus): number => { + const bucket = executionsByStatus.buckets.find((b: RawData) => b.key === status); + return Number(bucket?.doc_count || 0); + }; + + return { + total: Number(totalExecutions.value || 0), + by_outcome: { + succeeded: getStatusCount(RuleExecutionStatus.succeeded), + warning: getStatusCount(RuleExecutionStatus['partial failure']), + failed: getStatusCount(RuleExecutionStatus.failed), + }, + }; +}; + +const normalizeNumberOfLoggedMessages = ( + messageContainingEvents: RawData +): NumberOfLoggedMessages => { + const messagesByLogLevel = messageContainingEvents.messagesByLogLevel || {}; + + const getMessageCount = (level: LogLevel): number => { + const bucket = messagesByLogLevel.buckets.find((b: RawData) => b.key === level); + return Number(bucket?.doc_count || 0); + }; + + return { + total: Number(messageContainingEvents.doc_count || 0), + by_level: { + error: getMessageCount(LogLevel.error), + warn: getMessageCount(LogLevel.warn), + info: getMessageCount(LogLevel.info), + debug: getMessageCount(LogLevel.debug), + trace: getMessageCount(LogLevel.trace), + }, + }; +}; + +const normalizeNumberOfDetectedGaps = (gaps: RawData): NumberOfDetectedGaps => { + return { + total: Number(gaps.doc_count || 0), + total_duration_s: Number(gaps.totalGapDurationS?.value || 0), + }; +}; + +const normalizeAggregatedMetric = ( + percentilesAggregate: RawData, + modifier: (value: number) => number = (v) => v +): AggregatedMetric => { + const rawPercentiles = percentilesAggregate.values || {}; + return { + percentiles: mapValues(rawPercentiles, (rawValue) => modifier(Number(rawValue || 0))), + }; +}; + +const normalizeTopErrors = (messageContainingEvents: RawData): TopMessages => { + const topErrors = messageContainingEvents.errors?.topErrors || {}; + return normalizeTopMessages(topErrors); +}; + +const normalizeTopWarnings = (messageContainingEvents: RawData): TopMessages => { + const topWarnings = messageContainingEvents.warnings?.topWarnings || {}; + return normalizeTopMessages(topWarnings); +}; + +const normalizeTopMessages = (categorizeTextAggregate: RawData): TopMessages => { + const buckets = (categorizeTextAggregate || {}).buckets || []; + return buckets.map((b: RawData) => { + return { + count: Number(b?.doc_count || 0), + message: String(b?.key || ''), + }; + }); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/detection_engine_health/event_log/event_log_health_client.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/detection_engine_health/event_log/event_log_health_client.ts new file mode 100644 index 000000000000000..763ce36a4709906 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/detection_engine_health/event_log/event_log_health_client.ts @@ -0,0 +1,109 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { KueryNode } from '@kbn/es-query'; +import type { IEventLogClient } from '@kbn/event-log-plugin/server'; +import type { + ClusterHealthParameters, + ClusterHealthSnapshot, + RuleHealthParameters, + RuleHealthSnapshot, + SpaceHealthParameters, + SpaceHealthSnapshot, +} from '../../../../../../../common/detection_engine/rule_monitoring'; + +import * as f from '../../event_log/event_log_fields'; +import { + ALERTING_PROVIDER, + RULE_EXECUTION_LOG_PROVIDER, + RULE_SAVED_OBJECT_TYPE, +} from '../../event_log/event_log_constants'; +import { kqlOr } from '../../utils/kql'; +import { + getRuleHealthAggregation, + normalizeRuleHealthAggregationResult, +} from './aggregations/health_stats_for_rule'; + +/** + * Client for calculating health stats based on events in .kibana-event-log-* index. + */ +export interface IEventLogHealthClient { + calculateRuleHealth(args: RuleHealthParameters): Promise; + calculateSpaceHealth(args: SpaceHealthParameters): Promise; + calculateClusterHealth(args: ClusterHealthParameters): Promise; +} + +type RuleHealth = Omit; +type SpaceHealth = Omit; +type ClusterHealth = Omit; + +export const createEventLogHealthClient = (eventLog: IEventLogClient): IEventLogHealthClient => { + return { + async calculateRuleHealth(args: RuleHealthParameters): Promise { + const { rule_id: ruleId, interval } = args; + const soType = RULE_SAVED_OBJECT_TYPE; + const soIds = [ruleId]; + const eventProviders = [RULE_EXECUTION_LOG_PROVIDER, ALERTING_PROVIDER]; + + const kqlFilter = `${f.EVENT_PROVIDER}:${kqlOr(eventProviders)}`; + const aggs = getRuleHealthAggregation(interval.granularity); + + const result = await eventLog.aggregateEventsBySavedObjectIds(soType, soIds, { + start: interval.from, + end: interval.to, + filter: kqlFilter, + aggs, + }); + + return normalizeRuleHealthAggregationResult(result, aggs); + }, + + async calculateSpaceHealth(args: SpaceHealthParameters): Promise { + const { interval } = args; + const soType = RULE_SAVED_OBJECT_TYPE; + const authFilter = {} as KueryNode; + const namespaces = undefined; // means current Kibana space + const eventProviders = [RULE_EXECUTION_LOG_PROVIDER, ALERTING_PROVIDER]; + + const kqlFilter = `${f.EVENT_PROVIDER}:${kqlOr(eventProviders)}`; + const aggs = getRuleHealthAggregation(interval.granularity); + + // TODO: https://github.com/elastic/kibana/issues/125642 Check with ResponseOps that this is correct usage of this method + const result = await eventLog.aggregateEventsWithAuthFilter( + soType, + authFilter, + { + start: interval.from, + end: interval.to, + filter: kqlFilter, + aggs, + }, + namespaces + ); + + return normalizeRuleHealthAggregationResult(result, aggs); + }, + + async calculateClusterHealth(args: ClusterHealthParameters): Promise { + // TODO: https://github.com/elastic/kibana/issues/125642 Implement + return { + stats_over_interval: { + message: 'Not implemented', + }, + history_over_interval: { + buckets: [], + }, + debug: { + eventLog: { + request: {}, + response: {}, + }, + }, + }; + }, + }; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/detection_engine_health/index.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/detection_engine_health/index.ts new file mode 100644 index 000000000000000..5d0c8ed6743eaff --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/detection_engine_health/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './detection_engine_health_client_interface'; +export * from './detection_engine_health_client'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/detection_engine_health/rule_objects/aggregations/health_stats_for_space.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/detection_engine_health/rule_objects/aggregations/health_stats_for_space.ts new file mode 100644 index 000000000000000..9f3dc9eab9da3df --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/detection_engine_health/rule_objects/aggregations/health_stats_for_space.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import type { SpaceHealthStatsAtTheMoment } from '../../../../../../../../common/detection_engine/rule_monitoring'; +import { getRuleStatsAggregation, normalizeRuleStatsAggregation } from './rule_stats'; + +export const getSpaceHealthAggregation = (): Record< + string, + estypes.AggregationsAggregationContainer +> => { + return getRuleStatsAggregation(); +}; + +export const normalizeSpaceHealthAggregationResult = ( + aggregations: Record +): SpaceHealthStatsAtTheMoment => { + return normalizeRuleStatsAggregation(aggregations); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/detection_engine_health/rule_objects/aggregations/rule_stats.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/detection_engine_health/rule_objects/aggregations/rule_stats.ts new file mode 100644 index 000000000000000..569cd92f177c9df --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/detection_engine_health/rule_objects/aggregations/rule_stats.ts @@ -0,0 +1,112 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import type { + RuleStats, + TotalEnabledDisabled, +} from '../../../../../../../../common/detection_engine/rule_monitoring'; +import type { RawData } from '../../../utils/normalization'; + +export const getRuleStatsAggregation = (): Record< + string, + estypes.AggregationsAggregationContainer +> => { + const rulesByEnabled: estypes.AggregationsAggregationContainer = { + terms: { + field: 'alert.attributes.enabled', + }, + }; + + return { + rulesByEnabled, + rulesByOrigin: { + terms: { + field: 'alert.attributes.params.immutable', + }, + aggs: { + rulesByEnabled, + }, + }, + rulesByType: { + terms: { + field: 'alert.attributes.alertTypeId', + }, + aggs: { + rulesByEnabled, + }, + }, + rulesByOutcome: { + terms: { + field: 'alert.attributes.lastRun.outcome', + }, + aggs: { + rulesByEnabled, + }, + }, + }; +}; + +export const normalizeRuleStatsAggregation = (aggregations: Record): RuleStats => { + const rulesByEnabled = aggregations.rulesByEnabled || {}; + const rulesByOrigin = aggregations.rulesByOrigin || {}; + const rulesByType = aggregations.rulesByType || {}; + const rulesByOutcome = aggregations.rulesByOutcome || {}; + + return { + number_of_rules: { + all: normalizeByEnabled(rulesByEnabled), + by_origin: normalizeByOrigin(rulesByOrigin), + by_type: normalizeByAnyKeyword(rulesByType), + by_outcome: normalizeByAnyKeyword(rulesByOutcome), + }, + }; +}; + +const normalizeByEnabled = (rulesByEnabled: RawData): TotalEnabledDisabled => { + const getEnabled = (value: 'true' | 'false'): number => { + const bucket = rulesByEnabled?.buckets?.find((b: RawData) => b.key_as_string === value); + return Number(bucket?.doc_count || 0); + }; + + const enabled = getEnabled('true'); + const disabled = getEnabled('false'); + + return { + total: enabled + disabled, + enabled, + disabled, + }; +}; + +const normalizeByOrigin = ( + rulesByOrigin: RawData +): Record<'prebuilt' | 'custom', TotalEnabledDisabled> => { + const getOrigin = (value: 'true' | 'false'): TotalEnabledDisabled => { + const bucket = rulesByOrigin?.buckets?.find((b: RawData) => b.key === value); + return normalizeByEnabled(bucket?.rulesByEnabled); + }; + + const prebuilt = getOrigin('true'); + const custom = getOrigin('false'); + + return { prebuilt, custom }; +}; + +const normalizeByAnyKeyword = ( + rulesByAnyKeyword: RawData +): Record => { + const kvPairs = rulesByAnyKeyword?.buckets?.map((b: RawData) => { + const bucketKey = b.key; + const rulesByEnabled = b?.rulesByEnabled || {}; + return { + [bucketKey]: normalizeByEnabled(rulesByEnabled), + }; + }); + + return Object.assign({}, ...kvPairs); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/detection_engine_health/rule_objects/fetch_rule_by_id.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/detection_engine_health/rule_objects/fetch_rule_by_id.ts new file mode 100644 index 000000000000000..c33455827251e63 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/detection_engine_health/rule_objects/fetch_rule_by_id.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import Boom from '@hapi/boom'; +import type { RulesClient } from '@kbn/alerting-plugin/server'; +import type { + RuleObjectId, + RuleResponse, +} from '../../../../../../../common/detection_engine/rule_schema'; +import { readRules } from '../../../../rule_management/logic/crud/read_rules'; +import { transform } from '../../../../rule_management/utils/utils'; + +// TODO: https://github.com/elastic/kibana/issues/125642 Move to rule_management into a RuleManagementClient + +export const fetchRuleById = async ( + rulesClient: RulesClient, + id: RuleObjectId +): Promise => { + const rawRule = await readRules({ + id, + rulesClient, + ruleId: undefined, + }); + + if (rawRule == null) { + throw Boom.notFound(`Rule not found, id: "${id}" `); + } + + const normalizedRule = transform(rawRule); + + if (normalizedRule == null) { + throw Boom.internal('Internal error normalizing rule object'); + } + + return normalizedRule; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/detection_engine_health/rule_objects/rule_objects_health_client.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/detection_engine_health/rule_objects/rule_objects_health_client.ts new file mode 100644 index 000000000000000..624bd863335527b --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/detection_engine_health/rule_objects/rule_objects_health_client.ts @@ -0,0 +1,98 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { RulesClientApi } from '@kbn/alerting-plugin/server/types'; +import type { + ClusterHealthParameters, + ClusterHealthSnapshot, + RuleHealthParameters, + RuleHealthSnapshot, + SpaceHealthParameters, + SpaceHealthSnapshot, +} from '../../../../../../../common/detection_engine/rule_monitoring'; +import { + getSpaceHealthAggregation, + normalizeSpaceHealthAggregationResult, +} from './aggregations/health_stats_for_space'; +import { fetchRuleById } from './fetch_rule_by_id'; + +/** + * Client for calculating health stats based on rule saved objects. + */ +export interface IRuleObjectsHealthClient { + calculateRuleHealth(args: RuleHealthParameters): Promise; + calculateSpaceHealth(args: SpaceHealthParameters): Promise; + calculateClusterHealth(args: ClusterHealthParameters): Promise; +} + +type RuleHealth = Pick; +type SpaceHealth = Pick; +type ClusterHealth = Pick; + +export const createRuleObjectsHealthClient = ( + rulesClient: RulesClientApi +): IRuleObjectsHealthClient => { + return { + async calculateRuleHealth(args: RuleHealthParameters): Promise { + const rule = await fetchRuleById(rulesClient, args.rule_id); + return { + stats_at_the_moment: { rule }, + debug: {}, + }; + }, + + async calculateSpaceHealth(args: SpaceHealthParameters): Promise { + const aggs = getSpaceHealthAggregation(); + const aggregations = await rulesClient.aggregate({ aggs }); + + return { + stats_at_the_moment: normalizeSpaceHealthAggregationResult(aggregations), + debug: { + rulesClient: { + request: { aggs }, + response: { aggregations }, + }, + }, + }; + }, + + async calculateClusterHealth(args: ClusterHealthParameters): Promise { + // TODO: https://github.com/elastic/kibana/issues/125642 Implement + return { + stats_at_the_moment: { + number_of_rules: { + all: { + total: 0, + enabled: 0, + disabled: 0, + }, + by_origin: { + prebuilt: { + total: 0, + enabled: 0, + disabled: 0, + }, + custom: { + total: 0, + enabled: 0, + disabled: 0, + }, + }, + by_type: {}, + by_outcome: {}, + }, + }, + debug: { + rulesClient: { + request: {}, + response: {}, + }, + }, + }; + }, + }; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/event_log/constants.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/event_log/event_log_constants.ts similarity index 89% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/event_log/constants.ts rename to x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/event_log/event_log_constants.ts index 3493e49e88135b2..2aeb22fa796798d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/event_log/constants.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/event_log/event_log_constants.ts @@ -8,3 +8,5 @@ export const RULE_SAVED_OBJECT_TYPE = 'alert'; export const RULE_EXECUTION_LOG_PROVIDER = 'securitySolution.ruleExecution'; + +export const ALERTING_PROVIDER = 'alerting'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/event_log/event_log_fields.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/event_log/event_log_fields.ts new file mode 100644 index 000000000000000..4e66e93f5450738 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/event_log/event_log_fields.ts @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +// ------------------------------------------------------------------------------------------------- +// ECS fields + +export const TIMESTAMP = `@timestamp` as const; + +export const EVENT_PROVIDER = 'event.provider' as const; +export const EVENT_ACTION = 'event.action' as const; +export const EVENT_SEQUENCE = 'event.sequence' as const; + +export const LOG_LEVEL = 'log.level' as const; + +// ------------------------------------------------------------------------------------------------- +// Custom fields of Alerting Framework and Security Solution + +const RULE_EXECUTION = 'kibana.alert.rule.execution' as const; +const RULE_EXECUTION_METRICS = `${RULE_EXECUTION}.metrics` as const; + +export const RULE_EXECUTION_UUID = `${RULE_EXECUTION}.uuid` as const; + +export const RULE_EXECUTION_OUTCOME = 'kibana.alerting.outcome' as const; + +export const RULE_EXECUTION_STATUS = `${RULE_EXECUTION}.status` as const; + +export const RULE_EXECUTION_TOTAL_DURATION_MS = + `${RULE_EXECUTION_METRICS}.total_run_duration_ms` as const; + +export const RULE_EXECUTION_SEARCH_DURATION_MS = + `${RULE_EXECUTION_METRICS}.total_search_duration_ms` as const; + +export const RULE_EXECUTION_INDEXING_DURATION_MS = + `${RULE_EXECUTION_METRICS}.total_indexing_duration_ms` as const; + +export const RULE_EXECUTION_GAP_DURATION_S = + `${RULE_EXECUTION_METRICS}.execution_gap_duration_s` as const; + +export const RULE_EXECUTION_SCHEDULE_DELAY_NS = 'kibana.task.schedule_delay' as const; + +export const NUMBER_OF_ALERTS_GENERATED = `${RULE_EXECUTION_METRICS}.alert_counts.new` as const; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/event_log/register_event_log_provider.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/event_log/register_event_log_provider.ts similarity index 75% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/event_log/register_event_log_provider.ts rename to x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/event_log/register_event_log_provider.ts index 94542e913d45403..cec9aee84347ec6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/event_log/register_event_log_provider.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/event_log/register_event_log_provider.ts @@ -6,8 +6,8 @@ */ import type { IEventLogService } from '@kbn/event-log-plugin/server'; -import { RuleExecutionEventType } from '../../../../../../../common/detection_engine/rule_monitoring'; -import { RULE_EXECUTION_LOG_PROVIDER } from './constants'; +import { RuleExecutionEventType } from '../../../../../../common/detection_engine/rule_monitoring'; +import { RULE_EXECUTION_LOG_PROVIDER } from './event_log_constants'; export const registerEventLogProvider = (eventLogService: IEventLogService) => { eventLogService.registerProviderActions( diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/client_for_executors/client.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/client_for_executors/client.ts index a8d61aa7dda842a..6c1f407eef08ea0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/client_for_executors/client.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/client_for_executors/client.ts @@ -27,8 +27,8 @@ import { import { assertUnreachable } from '../../../../../../../common/utility_types'; import { withSecuritySpan } from '../../../../../../utils/with_security_span'; -import { truncateValue } from '../utils/normalization'; -import type { ExtMeta } from '../utils/console_logging'; +import { truncateValue } from '../../utils/normalization'; +import type { ExtMeta } from '../../utils/console_logging'; import { getCorrelationIds } from './correlation_ids'; import type { IEventLogWriter } from '../event_log/event_log_writer'; @@ -38,7 +38,7 @@ import type { StatusChangeArgs, } from './client_interface'; -export const createClientForExecutors = ( +export const createRuleExecutionLogClientForExecutors = ( settings: RuleExecutionSettings, eventLog: IEventLogWriter, logger: Logger, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/client_for_executors/correlation_ids.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/client_for_executors/correlation_ids.ts index 402635554a0e791..c74d762e150b851 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/client_for_executors/correlation_ids.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/client_for_executors/correlation_ids.ts @@ -5,7 +5,7 @@ * 2.0. */ -import type { ExtMeta } from '../utils/console_logging'; +import type { ExtMeta } from '../../utils/console_logging'; import type { RuleExecutionStatus } from '../../../../../../../common/detection_engine/rule_monitoring'; import type { RuleExecutionContext } from './client_interface'; @@ -75,7 +75,7 @@ const createBuilder = (state: BuilderState): ICorrelationIds => { }, }; - if (status != null && logMeta.rule.execution != null) { + if (status != null && logMeta.rule?.execution != null) { logMeta.rule.execution.status = status; } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/client_for_routes/client.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/client_for_routes/client.ts index f760eb9cd2455ff..78792722155fabe 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/client_for_routes/client.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/client_for_routes/client.ts @@ -6,22 +6,22 @@ */ import type { Logger } from '@kbn/core/server'; -import { withSecuritySpan } from '../../../../../../utils/with_security_span'; -import type { ExtMeta } from '../utils/console_logging'; import type { GetRuleExecutionEventsResponse, GetRuleExecutionResultsResponse, } from '../../../../../../../common/detection_engine/rule_monitoring'; +import { withSecuritySpan } from '../../../../../../utils/with_security_span'; import type { IEventLogReader } from '../event_log/event_log_reader'; +import type { ExtMeta } from '../../utils/console_logging'; import type { GetExecutionEventsArgs, GetExecutionResultsArgs, IRuleExecutionLogForRoutes, } from './client_interface'; -export const createClientForRoutes = ( +export const createRuleExecutionLogClientForRoutes = ( eventLog: IEventLogReader, logger: Logger ): IRuleExecutionLogForRoutes => { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/client_for_routes/client_interface.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/client_for_routes/client_interface.ts index a3fb5365181eba6..20e73f46e11bd84 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/client_for_routes/client_interface.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/client_for_routes/client_interface.ts @@ -5,7 +5,6 @@ * 2.0. */ -import type { SortOrder } from '../../../../../../../common/detection_engine/schemas/common'; import type { GetRuleExecutionEventsResponse, GetRuleExecutionResultsResponse, @@ -14,6 +13,8 @@ import type { RuleExecutionStatus, SortFieldOfRuleExecutionResult, } from '../../../../../../../common/detection_engine/rule_monitoring'; +import type { RuleObjectId } from '../../../../../../../common/detection_engine/rule_schema'; +import type { SortOrder } from '../../../../../../../common/detection_engine/schemas/common'; /** * Used from route handlers to fetch and manage various information about the rule execution: @@ -36,7 +37,7 @@ export interface IRuleExecutionLogForRoutes { export interface GetExecutionEventsArgs { /** Saved object id of the rule (`rule.id`). */ - ruleId: string; + ruleId: RuleObjectId; /** Include events of the specified types. If empty, all types of events will be included. */ eventTypes: RuleExecutionEventType[]; @@ -56,7 +57,7 @@ export interface GetExecutionEventsArgs { export interface GetExecutionResultsArgs { /** Saved object id of the rule (`rule.id`). */ - ruleId: string; + ruleId: RuleObjectId; /** Start of daterange to filter to. */ start: string; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/event_log/get_execution_event_aggregation/index.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/event_log/aggregations/execution_results/index.test.ts similarity index 99% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/event_log/get_execution_event_aggregation/index.test.ts rename to x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/event_log/aggregations/execution_results/index.test.ts index 83bf237747dda8e..f72630fba68fef3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/event_log/get_execution_event_aggregation/index.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/event_log/aggregations/execution_results/index.test.ts @@ -13,7 +13,7 @@ */ import { MAX_EXECUTION_EVENTS_DISPLAYED } from '@kbn/securitysolution-rules'; -import { RuleExecutionStatus } from '../../../../../../../../common/detection_engine/rule_monitoring'; +import { RuleExecutionStatus } from '../../../../../../../../../common/detection_engine/rule_monitoring'; import { formatExecutionEventResponse, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/event_log/get_execution_event_aggregation/index.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/event_log/aggregations/execution_results/index.ts similarity index 96% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/event_log/get_execution_event_aggregation/index.ts rename to x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/event_log/aggregations/execution_results/index.ts index c1ebb5e77f98a88..e12f2b3d8da5273 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/event_log/get_execution_event_aggregation/index.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/event_log/aggregations/execution_results/index.ts @@ -14,14 +14,16 @@ import type { AggregateEventsBySavedObjectResult } from '@kbn/event-log-plugin/s import type { RuleExecutionResult, GetRuleExecutionResultsResponse, -} from '../../../../../../../../common/detection_engine/rule_monitoring'; -import { RuleExecutionStatus } from '../../../../../../../../common/detection_engine/rule_monitoring'; +} from '../../../../../../../../../common/detection_engine/rule_monitoring'; +import { RuleExecutionStatus } from '../../../../../../../../../common/detection_engine/rule_monitoring'; import type { ExecutionEventAggregationOptions, ExecutionUuidAggResult, ExecutionUuidAggBucket, } from './types'; -import { EXECUTION_UUID_FIELD } from './types'; +import * as f from '../../../../event_log/event_log_fields'; + +// TODO: https://github.com/elastic/kibana/issues/125642 Move the fields from this file to `event_log_fields.ts` // Base ECS fields const ACTION_FIELD = 'event.action'; @@ -104,13 +106,13 @@ export const getExecutionEventAggregation = ({ // Total unique executions for given root filters totalExecutions: { cardinality: { - field: EXECUTION_UUID_FIELD, + field: f.RULE_EXECUTION_UUID, }, }, executionUuid: { // Bucket by execution UUID terms: { - field: EXECUTION_UUID_FIELD, + field: f.RULE_EXECUTION_UUID, size: maxExecutions, order: formatSortForTermsSort(sort), }, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/event_log/get_execution_event_aggregation/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/event_log/aggregations/execution_results/types.ts similarity index 94% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/event_log/get_execution_event_aggregation/types.ts rename to x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/event_log/aggregations/execution_results/types.ts index d98b5f5fc80cc6d..429da6eeac8bc5a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/event_log/get_execution_event_aggregation/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/event_log/aggregations/execution_results/types.ts @@ -7,9 +7,6 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -// Shared constants, consider moving to packages -export const EXECUTION_UUID_FIELD = 'kibana.alert.rule.execution.uuid'; - type AlertCounts = estypes.AggregationsMultiBucketAggregateBase & { buckets: { activeAlerts: estypes.AggregationsSingleBucketAggregateBase; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/event_log/event_log_reader.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/event_log/event_log_reader.ts index 2428274165b9c96..edebe65f2802341 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/event_log/event_log_reader.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/event_log/event_log_reader.ts @@ -9,14 +9,10 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import type { IEventLogClient, IValidatedEvent } from '@kbn/event-log-plugin/server'; import { MAX_EXECUTION_EVENTS_DISPLAYED } from '@kbn/securitysolution-rules'; -import { assertUnreachable } from '../../../../../../../common/utility_types'; -import { invariant } from '../../../../../../../common/utils/invariant'; -import { withSecuritySpan } from '../../../../../../utils/with_security_span'; - import type { - RuleExecutionEvent, GetRuleExecutionEventsResponse, GetRuleExecutionResultsResponse, + RuleExecutionEvent, } from '../../../../../../../common/detection_engine/rule_monitoring'; import { LogLevel, @@ -24,19 +20,28 @@ import { RuleExecutionEventType, ruleExecutionEventTypeFromString, } from '../../../../../../../common/detection_engine/rule_monitoring'; + +import { assertUnreachable } from '../../../../../../../common/utility_types'; +import { invariant } from '../../../../../../../common/utils/invariant'; +import { withSecuritySpan } from '../../../../../../utils/with_security_span'; +import { kqlAnd, kqlOr } from '../../utils/kql'; + import type { GetExecutionEventsArgs, GetExecutionResultsArgs, } from '../client_for_routes/client_interface'; - -import { RULE_SAVED_OBJECT_TYPE, RULE_EXECUTION_LOG_PROVIDER } from './constants'; import { formatExecutionEventResponse, getExecutionEventAggregation, mapRuleExecutionStatusToPlatformStatus, -} from './get_execution_event_aggregation'; -import type { ExecutionUuidAggResult } from './get_execution_event_aggregation/types'; -import { EXECUTION_UUID_FIELD } from './get_execution_event_aggregation/types'; +} from './aggregations/execution_results'; +import type { ExecutionUuidAggResult } from './aggregations/execution_results/types'; + +import * as f from '../../event_log/event_log_fields'; +import { + RULE_EXECUTION_LOG_PROVIDER, + RULE_SAVED_OBJECT_TYPE, +} from '../../event_log/event_log_constants'; export interface IEventLogReader { getExecutionEvents(args: GetExecutionEventsArgs): Promise; @@ -54,17 +59,17 @@ export const createEventLogReader = (eventLog: IEventLogClient): IEventLogReader // TODO: include Framework events const kqlFilter = kqlAnd([ - `event.provider:${RULE_EXECUTION_LOG_PROVIDER}`, - eventTypes.length > 0 ? `event.action:(${kqlOr(eventTypes)})` : '', - logLevels.length > 0 ? `log.level:(${kqlOr(logLevels)})` : '', + `${f.EVENT_PROVIDER}:${RULE_EXECUTION_LOG_PROVIDER}`, + eventTypes.length > 0 ? `${f.EVENT_ACTION}:(${kqlOr(eventTypes)})` : '', + logLevels.length > 0 ? `${f.LOG_LEVEL}:(${kqlOr(logLevels)})` : '', ]); const findResult = await withSecuritySpan('findEventsBySavedObjectIds', () => { return eventLog.findEventsBySavedObjectIds(soType, soIds, { filter: kqlFilter, sort: [ - { sort_field: '@timestamp', sort_order: sortOrder }, - { sort_field: 'event.sequence', sort_order: sortOrder }, + { sort_field: f.TIMESTAMP, sort_order: sortOrder }, + { sort_field: f.EVENT_SEQUENCE, sort_order: sortOrder }, ], page, per_page: perPage, @@ -102,25 +107,23 @@ export const createEventLogReader = (eventLog: IEventLogClient): IEventLogReader start, end, // Also query for `event.outcome` to catch executions that only contain platform events - filter: `kibana.alert.rule.execution.status:(${statusFilters.join( - ' OR ' - )}) ${outcomeFilter}`, + filter: `${f.RULE_EXECUTION_STATUS}:(${statusFilters.join(' OR ')}) ${outcomeFilter}`, aggs: { totalExecutions: { cardinality: { - field: EXECUTION_UUID_FIELD, + field: f.RULE_EXECUTION_UUID, }, }, filteredExecutionUUIDs: { terms: { - field: EXECUTION_UUID_FIELD, + field: f.RULE_EXECUTION_UUID, order: { executeStartTime: 'desc' }, size: MAX_EXECUTION_EVENTS_DISPLAYED, }, aggs: { executeStartTime: { min: { - field: '@timestamp', + field: f.TIMESTAMP, }, }, }, @@ -144,7 +147,7 @@ export const createEventLogReader = (eventLog: IEventLogClient): IEventLogReader // Now query for aggregate events, and pass any ID's as filters as determined from the above status/queryText results const idsFilter = statusIds.length - ? `kibana.alert.rule.execution.uuid:(${statusIds.join(' OR ')})` + ? `${f.RULE_EXECUTION_UUID}:(${statusIds.join(' OR ')})` : ''; const results = await eventLog.aggregateEventsBySavedObjectIds(soType, soIds, { start, @@ -163,14 +166,6 @@ export const createEventLogReader = (eventLog: IEventLogClient): IEventLogReader }; }; -const kqlAnd = (items: T[]): string => { - return items.filter(Boolean).map(String).join(' and '); -}; - -const kqlOr = (items: T[]): string => { - return items.filter(Boolean).map(String).join(' or '); -}; - const normalizeEvent = (rawEvent: IValidatedEvent): RuleExecutionEvent => { invariant(rawEvent, 'Event not found'); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/event_log/event_log_writer.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/event_log/event_log_writer.ts index ceed2f1d3a739af..e9fffaca8e91616 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/event_log/event_log_writer.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/event_log/event_log_writer.ts @@ -19,7 +19,10 @@ import { RuleExecutionEventType, ruleExecutionStatusToNumber, } from '../../../../../../../common/detection_engine/rule_monitoring'; -import { RULE_SAVED_OBJECT_TYPE, RULE_EXECUTION_LOG_PROVIDER } from './constants'; +import { + RULE_SAVED_OBJECT_TYPE, + RULE_EXECUTION_LOG_PROVIDER, +} from '../../event_log/event_log_constants'; export interface IEventLogWriter { logMessage(args: MessageArgs): void; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/index.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/index.ts index 40f9babb53af5df..a3309b270de21d0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/index.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/index.ts @@ -7,9 +7,5 @@ export * from './client_for_executors/client_interface'; export * from './client_for_routes/client_interface'; -export * from './service_interface'; -export * from './service'; -export { RULE_EXECUTION_LOG_PROVIDER } from './event_log/constants'; export { createRuleExecutionSummary } from './create_rule_execution_summary'; -export * from './utils/normalization'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/service.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/service.ts deleted file mode 100644 index cea7f5d23a14f68..000000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/service.ts +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { Logger } from '@kbn/core/server'; -import { invariant } from '../../../../../../common/utils/invariant'; -import type { ConfigType } from '../../../../../config'; -import { withSecuritySpan } from '../../../../../utils/with_security_span'; -import type { - SecuritySolutionPluginCoreSetupDependencies, - SecuritySolutionPluginSetupDependencies, -} from '../../../../../plugin_contract'; - -import type { IRuleExecutionLogForRoutes } from './client_for_routes/client_interface'; -import { createClientForRoutes } from './client_for_routes/client'; -import type { IRuleExecutionLogForExecutors } from './client_for_executors/client_interface'; -import { createClientForExecutors } from './client_for_executors/client'; - -import { registerEventLogProvider } from './event_log/register_event_log_provider'; -import { createEventLogReader } from './event_log/event_log_reader'; -import { createEventLogWriter } from './event_log/event_log_writer'; -import { fetchRuleExecutionSettings } from './execution_settings/fetch_rule_execution_settings'; -import type { - ClientForExecutorsParams, - ClientForRoutesParams, - IRuleExecutionLogService, -} from './service_interface'; - -export const createRuleExecutionLogService = ( - config: ConfigType, - logger: Logger, - core: SecuritySolutionPluginCoreSetupDependencies, - plugins: SecuritySolutionPluginSetupDependencies -): IRuleExecutionLogService => { - return { - registerEventLogProvider: () => { - registerEventLogProvider(plugins.eventLog); - }, - - createClientForRoutes: (params: ClientForRoutesParams): IRuleExecutionLogForRoutes => { - const { eventLogClient } = params; - - const eventLogReader = createEventLogReader(eventLogClient); - - return createClientForRoutes(eventLogReader, logger); - }, - - createClientForExecutors: ( - params: ClientForExecutorsParams - ): Promise => { - return withSecuritySpan('IRuleExecutionLogService.createClientForExecutors', async () => { - const { savedObjectsClient, context, ruleMonitoringService, ruleResultService } = params; - - invariant(ruleMonitoringService, 'ruleMonitoringService required for detection rules'); - invariant(ruleResultService, 'ruleResultService required for detection rules'); - - const childLogger = logger.get('ruleExecution'); - - const ruleExecutionSettings = await fetchRuleExecutionSettings( - config, - childLogger, - core, - savedObjectsClient - ); - - const eventLogWriter = createEventLogWriter(plugins.eventLog); - - return createClientForExecutors( - ruleExecutionSettings, - eventLogWriter, - childLogger, - context, - ruleMonitoringService, - ruleResultService - ); - }); - }, - }; -}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/service_interface.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/service_interface.ts deleted file mode 100644 index aa8d5e0bfee36f4..000000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/service_interface.ts +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { SavedObjectsClientContract } from '@kbn/core/server'; -import type { IEventLogClient } from '@kbn/event-log-plugin/server'; - -import type { - PublicRuleResultService, - PublicRuleMonitoringService, -} from '@kbn/alerting-plugin/server/types'; -import type { IRuleExecutionLogForRoutes } from './client_for_routes/client_interface'; -import type { - IRuleExecutionLogForExecutors, - RuleExecutionContext, -} from './client_for_executors/client_interface'; - -export interface IRuleExecutionLogService { - registerEventLogProvider(): void; - - createClientForRoutes(params: ClientForRoutesParams): IRuleExecutionLogForRoutes; - - createClientForExecutors( - params: ClientForExecutorsParams - ): Promise; -} - -export interface ClientForRoutesParams { - savedObjectsClient: SavedObjectsClientContract; - eventLogClient: IEventLogClient; -} - -export interface ClientForExecutorsParams { - savedObjectsClient: SavedObjectsClientContract; - ruleMonitoringService?: PublicRuleMonitoringService; - ruleResultService?: PublicRuleResultService; - context: RuleExecutionContext; -} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/service.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/service.ts new file mode 100644 index 000000000000000..65f637ab7cef2a9 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/service.ts @@ -0,0 +1,104 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Logger } from '@kbn/core/server'; +import { invariant } from '../../../../../common/utils/invariant'; +import type { ConfigType } from '../../../../config'; +import { withSecuritySpan } from '../../../../utils/with_security_span'; +import type { + SecuritySolutionPluginCoreSetupDependencies, + SecuritySolutionPluginSetupDependencies, +} from '../../../../plugin_contract'; + +import type { IDetectionEngineHealthClient } from './detection_engine_health/detection_engine_health_client_interface'; +import type { IRuleExecutionLogForRoutes } from './rule_execution_log/client_for_routes/client_interface'; +import { createRuleExecutionLogClientForRoutes } from './rule_execution_log/client_for_routes/client'; +import type { IRuleExecutionLogForExecutors } from './rule_execution_log/client_for_executors/client_interface'; +import { createRuleExecutionLogClientForExecutors } from './rule_execution_log/client_for_executors/client'; + +import { registerEventLogProvider } from './event_log/register_event_log_provider'; +import { createDetectionEngineHealthClient } from './detection_engine_health/detection_engine_health_client'; +import { createEventLogHealthClient } from './detection_engine_health/event_log/event_log_health_client'; +import { createRuleObjectsHealthClient } from './detection_engine_health/rule_objects/rule_objects_health_client'; +import { createEventLogReader } from './rule_execution_log/event_log/event_log_reader'; +import { createEventLogWriter } from './rule_execution_log/event_log/event_log_writer'; +import { fetchRuleExecutionSettings } from './rule_execution_log/execution_settings/fetch_rule_execution_settings'; +import type { + RuleExecutionLogClientForExecutorsParams, + RuleExecutionLogClientForRoutesParams, + IRuleMonitoringService, + DetectionEngineHealthClientParams, +} from './service_interface'; + +export const createRuleMonitoringService = ( + config: ConfigType, + logger: Logger, + core: SecuritySolutionPluginCoreSetupDependencies, + plugins: SecuritySolutionPluginSetupDependencies +): IRuleMonitoringService => { + return { + registerEventLogProvider: () => { + registerEventLogProvider(plugins.eventLog); + }, + + createDetectionEngineHealthClient: ( + params: DetectionEngineHealthClientParams + ): IDetectionEngineHealthClient => { + const { rulesClient, eventLogClient, currentSpaceId } = params; + const ruleObjectsHealthClient = createRuleObjectsHealthClient(rulesClient); + const eventLogHealthClient = createEventLogHealthClient(eventLogClient); + return createDetectionEngineHealthClient( + ruleObjectsHealthClient, + eventLogHealthClient, + logger, + currentSpaceId + ); + }, + + createRuleExecutionLogClientForRoutes: ( + params: RuleExecutionLogClientForRoutesParams + ): IRuleExecutionLogForRoutes => { + const { eventLogClient } = params; + const eventLogReader = createEventLogReader(eventLogClient); + return createRuleExecutionLogClientForRoutes(eventLogReader, logger); + }, + + createRuleExecutionLogClientForExecutors: ( + params: RuleExecutionLogClientForExecutorsParams + ): Promise => { + return withSecuritySpan( + 'IRuleMonitoringService.createRuleExecutionLogClientForExecutors', + async () => { + const { savedObjectsClient, context, ruleMonitoringService, ruleResultService } = params; + + invariant(ruleMonitoringService, 'ruleMonitoringService required for detection rules'); + invariant(ruleResultService, 'ruleResultService required for detection rules'); + + const childLogger = logger.get('ruleExecution'); + + const ruleExecutionSettings = await fetchRuleExecutionSettings( + config, + childLogger, + core, + savedObjectsClient + ); + + const eventLogWriter = createEventLogWriter(plugins.eventLog); + + return createRuleExecutionLogClientForExecutors( + ruleExecutionSettings, + eventLogWriter, + childLogger, + context, + ruleMonitoringService, + ruleResultService + ); + } + ); + }, + }; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/service_interface.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/service_interface.ts new file mode 100644 index 000000000000000..14ee470aaae1241 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/service_interface.ts @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObjectsClientContract } from '@kbn/core/server'; +import type { IEventLogClient } from '@kbn/event-log-plugin/server'; +import type { + PublicRuleResultService, + PublicRuleMonitoringService, + RulesClientApi, +} from '@kbn/alerting-plugin/server/types'; + +import type { IDetectionEngineHealthClient } from './detection_engine_health/detection_engine_health_client_interface'; +import type { IRuleExecutionLogForRoutes } from './rule_execution_log/client_for_routes/client_interface'; +import type { + IRuleExecutionLogForExecutors, + RuleExecutionContext, +} from './rule_execution_log/client_for_executors/client_interface'; + +export interface IRuleMonitoringService { + registerEventLogProvider(): void; + + createDetectionEngineHealthClient( + params: DetectionEngineHealthClientParams + ): IDetectionEngineHealthClient; + + createRuleExecutionLogClientForRoutes( + params: RuleExecutionLogClientForRoutesParams + ): IRuleExecutionLogForRoutes; + + createRuleExecutionLogClientForExecutors( + params: RuleExecutionLogClientForExecutorsParams + ): Promise; +} + +export interface DetectionEngineHealthClientParams { + savedObjectsClient: SavedObjectsClientContract; + rulesClient: RulesClientApi; + eventLogClient: IEventLogClient; + currentSpaceId: string; +} + +export interface RuleExecutionLogClientForRoutesParams { + savedObjectsClient: SavedObjectsClientContract; + eventLogClient: IEventLogClient; +} + +export interface RuleExecutionLogClientForExecutorsParams { + savedObjectsClient: SavedObjectsClientContract; + ruleMonitoringService?: PublicRuleMonitoringService; + ruleResultService?: PublicRuleResultService; + context: RuleExecutionContext; +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/utils/console_logging.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/utils/console_logging.ts similarity index 84% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/utils/console_logging.ts rename to x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/utils/console_logging.ts index d45c5ee7c65d07e..aeb63125e64f0d3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/utils/console_logging.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/utils/console_logging.ts @@ -6,13 +6,13 @@ */ import type { LogMeta } from '@kbn/core/server'; -import type { RuleExecutionStatus } from '../../../../../../../common/detection_engine/rule_monitoring'; +import type { RuleExecutionStatus } from '../../../../../../common/detection_engine/rule_monitoring'; /** * Extended metadata that rule execution logger can attach to every console log record. */ export interface ExtMeta extends LogMeta { - rule: ExtRule; + rule?: ExtRule; kibana?: ExtKibana; } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/utils/es_aggregations.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/utils/es_aggregations.ts new file mode 100644 index 000000000000000..4056e88dc8eb143 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/utils/es_aggregations.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const DEFAULT_PERCENTILES: number[] = [50, 95, 99, 99.9]; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/utils/kql.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/utils/kql.ts new file mode 100644 index 000000000000000..0c60f7bec0bfa38 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/utils/kql.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const kqlAnd = (items: T[]): string => { + return items.filter(Boolean).map(String).join(' and '); +}; + +export const kqlOr = (items: T[]): string => { + return items.filter(Boolean).map(String).join(' or '); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/utils/normalization.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/utils/normalization.ts similarity index 90% rename from x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/utils/normalization.ts rename to x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/utils/normalization.ts index e24f63181116854..f3707ae738208cb 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/rule_execution_log/utils/normalization.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/utils/normalization.ts @@ -7,6 +7,12 @@ import { take, toString, truncate, uniq } from 'lodash'; +/** + * Useful for normalizing responses from Elasticsearch. + */ +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export type RawData = any; + // When we write rule execution status updates to saved objects or to event log, // we can write warning/failure messages as well. In some cases those messages // are built from N errors collected during the "big loop" of Detection Engine, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/mocks.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/mocks.ts index 73288c05c3a71cc..6c03f3e1d43c2d9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/mocks.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/mocks.ts @@ -5,4 +5,5 @@ * 2.0. */ +export * from './logic/detection_engine_health/__mocks__'; export * from './logic/rule_execution_log/__mocks__'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_preview/api/preview_rules/preview_rule_execution_logger.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_preview/api/preview_rules/preview_rule_execution_logger.ts index 46cb0e8adfbf4f3..d2a4c0aba9e0bbd 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_preview/api/preview_rules/preview_rule_execution_logger.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_preview/api/preview_rules/preview_rule_execution_logger.ts @@ -6,13 +6,13 @@ */ import type { - IRuleExecutionLogService, + IRuleMonitoringService, RuleExecutionContext, StatusChangeArgs, } from '../../../rule_monitoring'; export interface IPreviewRuleExecutionLogger { - factory: IRuleExecutionLogService['createClientForExecutors']; + factory: IRuleMonitoringService['createRuleExecutionLogClientForExecutors']; } export const createPreviewRuleExecutionLogger = ( diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts index 2a3eb82413940e1..0d20058379742c5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts @@ -40,7 +40,7 @@ import type { SetupPlugins } from '../../../plugin'; import type { CompleteRule, RuleParams } from '../rule_schema'; import type { ExperimentalFeatures } from '../../../../common/experimental_features'; import type { ITelemetryEventsSender } from '../../telemetry/sender'; -import type { IRuleExecutionLogForExecutors, IRuleExecutionLogService } from '../rule_monitoring'; +import type { IRuleExecutionLogForExecutors, IRuleMonitoringService } from '../rule_monitoring'; import type { RefreshTypes } from '../types'; import type { Status } from '../../../../common/detection_engine/schemas/common/schemas'; @@ -133,7 +133,7 @@ export interface CreateSecurityRuleTypeWrapperProps { config: ConfigType; publicBaseUrl: string | undefined; ruleDataClient: IRuleDataClient; - ruleExecutionLoggerFactory: IRuleExecutionLogService['createClientForExecutors']; + ruleExecutionLoggerFactory: IRuleMonitoringService['createRuleExecutionLogClientForExecutors']; version: string; isPreview?: boolean; } diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index b30b84977c3ed31..c9773fe576912cb 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -71,7 +71,7 @@ import { TelemetryReceiver } from './lib/telemetry/receiver'; import { licenseService } from './lib/license'; import { PolicyWatcher } from './endpoint/lib/policy/license_watch'; import previewPolicy from './lib/detection_engine/routes/index/preview_policy.json'; -import { createRuleExecutionLogService } from './lib/detection_engine/rule_monitoring'; +import { createRuleMonitoringService } from './lib/detection_engine/rule_monitoring'; import { getKibanaPrivilegesFeaturePrivileges, getCasesKibanaFeature } from './features'; import { EndpointMetadataService } from './endpoint/services/metadata'; import type { @@ -161,11 +161,13 @@ export class Plugin implements ISecuritySolutionPlugin { initSavedObjects(core.savedObjects); initUiSettings(core.uiSettings, experimentalFeatures); + if (experimentalFeatures.assistantEnabled ?? false) { plugins.actions.registerSubActionConnectorType(getGenerativeAiConnectorType()); } - const ruleExecutionLogService = createRuleExecutionLogService(config, logger, core, plugins); - ruleExecutionLogService.registerEventLogProvider(); + + const ruleMonitoringService = createRuleMonitoringService(config, logger, core, plugins); + ruleMonitoringService.registerEventLogProvider(); const requestContextFactory = new RequestContextFactory({ config, @@ -173,7 +175,7 @@ export class Plugin implements ISecuritySolutionPlugin { core, plugins, endpointAppContextService: this.endpointAppContextService, - ruleExecutionLogService, + ruleMonitoringService, kibanaVersion: pluginContext.env.packageInfo.version, kibanaBranch: pluginContext.env.packageInfo.branch, }); @@ -243,7 +245,7 @@ export class Plugin implements ISecuritySolutionPlugin { config: this.config, publicBaseUrl: core.http.basePath.publicBaseUrl, ruleDataClient, - ruleExecutionLoggerFactory: ruleExecutionLogService.createClientForExecutors, + ruleExecutionLoggerFactory: ruleMonitoringService.createRuleExecutionLogClientForExecutors, version: pluginContext.env.packageInfo.version, }; diff --git a/x-pack/plugins/security_solution/server/request_context_factory.ts b/x-pack/plugins/security_solution/server/request_context_factory.ts index 3bb125cf0691410..dbd36258260afb1 100644 --- a/x-pack/plugins/security_solution/server/request_context_factory.ts +++ b/x-pack/plugins/security_solution/server/request_context_factory.ts @@ -12,7 +12,7 @@ import type { Logger, KibanaRequest, RequestHandlerContext } from '@kbn/core/ser import { DEFAULT_SPACE_ID } from '../common/constants'; import { AppClientFactory } from './client'; import type { ConfigType } from './config'; -import type { IRuleExecutionLogService } from './lib/detection_engine/rule_monitoring'; +import type { IRuleMonitoringService } from './lib/detection_engine/rule_monitoring'; import { buildFrameworkRequest } from './lib/timeline/utils/common'; import type { SecuritySolutionPluginCoreSetupDependencies, @@ -39,7 +39,7 @@ interface ConstructorOptions { core: SecuritySolutionPluginCoreSetupDependencies; plugins: SecuritySolutionPluginSetupDependencies; endpointAppContextService: EndpointAppContextService; - ruleExecutionLogService: IRuleExecutionLogService; + ruleMonitoringService: IRuleMonitoringService; kibanaVersion: string; kibanaBranch: string; } @@ -56,13 +56,16 @@ export class RequestContextFactory implements IRequestContextFactory { request: KibanaRequest ): Promise { const { options, appClientFactory } = this; - const { config, core, plugins, endpointAppContextService, ruleExecutionLogService } = options; + const { config, core, plugins, endpointAppContextService, ruleMonitoringService } = options; const { lists, ruleRegistry, security } = plugins; const [, startPlugins] = await core.getStartServices(); const frameworkRequest = await buildFrameworkRequest(context, security, request); const coreContext = await context.core; + const getSpaceId = (): string => + startPlugins.spaces?.spacesService?.getSpaceId(request) || DEFAULT_SPACE_ID; + appClientFactory.setup({ getSpaceId: startPlugins.spaces?.spacesService?.getSpaceId, config, @@ -92,14 +95,23 @@ export class RequestContextFactory implements IRequestContextFactory { getAppClient: () => appClientFactory.create(request), - getSpaceId: () => startPlugins.spaces?.spacesService?.getSpaceId(request) || DEFAULT_SPACE_ID, + getSpaceId, getRuleDataService: () => ruleRegistry.ruleDataService, getRacClient: startPlugins.ruleRegistry.getRacClientWithRequest, + getDetectionEngineHealthClient: memoize(() => + ruleMonitoringService.createDetectionEngineHealthClient({ + savedObjectsClient: coreContext.savedObjects.client, + rulesClient: startPlugins.alerting.getRulesClientWithRequest(request), + eventLogClient: startPlugins.eventLog.getClient(request), + currentSpaceId: getSpaceId(), + }) + ), + getRuleExecutionLog: memoize(() => - ruleExecutionLogService.createClientForRoutes({ + ruleMonitoringService.createRuleExecutionLogClientForRoutes({ savedObjectsClient: coreContext.savedObjects.client, eventLogClient: startPlugins.eventLog.getClient(request), }) diff --git a/x-pack/plugins/security_solution/server/types.ts b/x-pack/plugins/security_solution/server/types.ts index 993d031dec440bf..859a60c07d2a210 100644 --- a/x-pack/plugins/security_solution/server/types.ts +++ b/x-pack/plugins/security_solution/server/types.ts @@ -21,7 +21,10 @@ import type { IRuleDataService, AlertsClient } from '@kbn/rule-registry-plugin/s import type { Immutable } from '../common/endpoint/types'; import { AppClient } from './client'; import type { ConfigType } from './config'; -import type { IRuleExecutionLogForRoutes } from './lib/detection_engine/rule_monitoring'; +import type { + IDetectionEngineHealthClient, + IRuleExecutionLogForRoutes, +} from './lib/detection_engine/rule_monitoring'; import type { FrameworkRequest } from './lib/framework'; import type { EndpointAuthz } from '../common/endpoint/types/authz'; import type { EndpointInternalFleetServicesInterface } from './endpoint/services/fleet'; @@ -36,6 +39,7 @@ export interface SecuritySolutionApiRequestHandlerContext { getAppClient: () => AppClient; getSpaceId: () => string; getRuleDataService: () => IRuleDataService; + getDetectionEngineHealthClient: () => IDetectionEngineHealthClient; getRuleExecutionLog: () => IRuleExecutionLogForRoutes; getRacClient: (req: KibanaRequest) => Promise; getExceptionListClient: () => ExceptionListClient | null;