[7.x] [Monitoring] Thread pool rejections alert (#79433)

packages/kbn-optimizer/limits.yml

@@ -54,7 +54,7 @@ pageLoadAssetSize:
  mapsLegacy: 116961
  mapsLegacyLicensing: 20214
  ml: 82187
- monitoring: 268758
+ monitoring: 50000
  navigation: 37413
  newsfeed: 42228
  observability: 89709

x-pack/plugins/monitoring/common/constants.ts

@@ -4,6 +4,10 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
+import { i18n } from '@kbn/i18n';
+import { CommonAlertParamDetail } from './types/alerts';
+import { AlertParamType } from './enums';
+
 /**
  * Helper string to add as a tag in every logging call
  */
@@ -215,15 +219,6 @@ export const REPORTING_SYSTEM_ID = 'reporting';
  */
 export const TELEMETRY_COLLECTION_INTERVAL = 86400000;
 
-/**
- * We want to slowly rollout the migration from watcher-based cluster alerts to
- * kibana alerts and we only want to enable the kibana alerts once all
- * watcher-based cluster alerts have been migrated so this flag will serve
- * as the only way to see the new UI and actually run Kibana alerts. It will
- * be false until all alerts have been migrated, then it will be removed
- */
-export const KIBANA_CLUSTER_ALERTS_ENABLED = false;
-
 /**
  * The prefix for all alert types used by monitoring
  */
@@ -238,6 +233,168 @@ export const ALERT_KIBANA_VERSION_MISMATCH = `${ALERT_PREFIX}alert_kibana_versio
 export const ALERT_LOGSTASH_VERSION_MISMATCH = `${ALERT_PREFIX}alert_logstash_version_mismatch`;
 export const ALERT_MEMORY_USAGE = `${ALERT_PREFIX}alert_jvm_memory_usage`;
 export const ALERT_MISSING_MONITORING_DATA = `${ALERT_PREFIX}alert_missing_monitoring_data`;
+export const ALERT_THREAD_POOL_SEARCH_REJECTIONS = `${ALERT_PREFIX}alert_thread_pool_search_rejections`;
+export const ALERT_THREAD_POOL_WRITE_REJECTIONS = `${ALERT_PREFIX}alert_thread_pool_write_rejections`;
+
+/**
+ * Legacy alerts details/label for server and public use
+ */
+export const LEGACY_ALERT_DETAILS = {
+ [ALERT_CLUSTER_HEALTH]: {
+ label: i18n.translate('xpack.monitoring.alerts.clusterHealth.label', {
+ defaultMessage: 'Cluster health',
+ }),
+ },
+ [ALERT_ELASTICSEARCH_VERSION_MISMATCH]: {
+ label: i18n.translate('xpack.monitoring.alerts.elasticsearchVersionMismatch.label', {
+ defaultMessage: 'Elasticsearch version mismatch',
+ }),
+ },
+ [ALERT_KIBANA_VERSION_MISMATCH]: {
+ label: i18n.translate('xpack.monitoring.alerts.kibanaVersionMismatch.label', {
+ defaultMessage: 'Kibana version mismatch',
+ }),
+ },
+ [ALERT_LICENSE_EXPIRATION]: {
+ label: i18n.translate('xpack.monitoring.alerts.licenseExpiration.label', {
+ defaultMessage: 'License expiration',
+ }),
+ },
+ [ALERT_LOGSTASH_VERSION_MISMATCH]: {
+ label: i18n.translate('xpack.monitoring.alerts.logstashVersionMismatch.label', {
+ defaultMessage: 'Logstash version mismatch',
+ }),
+ },
+ [ALERT_NODES_CHANGED]: {
+ label: i18n.translate('xpack.monitoring.alerts.nodesChanged.label', {
+ defaultMessage: 'Nodes changed',
+ }),
+ },
+};
+
+/**
+ * Alerts details/label for server and public use
+ */
+export const ALERT_DETAILS = {
+ [ALERT_CPU_USAGE]: {
+ label: i18n.translate('xpack.monitoring.alerts.cpuUsage.label', {
+ defaultMessage: 'CPU Usage',
+ }),
+ paramDetails: {
+ threshold: {
+ label: i18n.translate('xpack.monitoring.alerts.cpuUsage.paramDetails.threshold.label', {
+ defaultMessage: `Notify when CPU is over`,
+ }),
+ type: AlertParamType.Percentage,
+ } as CommonAlertParamDetail,
+ duration: {
+ label: i18n.translate('xpack.monitoring.alerts.cpuUsage.paramDetails.duration.label', {
+ defaultMessage: `Look at the average over`,
+ }),
+ type: AlertParamType.Duration,
+ } as CommonAlertParamDetail,
+ },
+ },
+ [ALERT_DISK_USAGE]: {
+ paramDetails: {
+ threshold: {
+ label: i18n.translate('xpack.monitoring.alerts.diskUsage.paramDetails.threshold.label', {
+ defaultMessage: `Notify when disk capacity is over`,
+ }),
+ type: AlertParamType.Percentage,
+ },
+ duration: {
+ label: i18n.translate('xpack.monitoring.alerts.diskUsage.paramDetails.duration.label', {
+ defaultMessage: `Look at the average over`,
+ }),
+ type: AlertParamType.Duration,
+ },
+ },
+ label: i18n.translate('xpack.monitoring.alerts.diskUsage.label', {
+ defaultMessage: 'Disk Usage',
+ }),
+ },
+ [ALERT_MEMORY_USAGE]: {
+ paramDetails: {
+ threshold: {
+ label: i18n.translate('xpack.monitoring.alerts.memoryUsage.paramDetails.threshold.label', {
+ defaultMessage: `Notify when memory usage is over`,
+ }),
+ type: AlertParamType.Percentage,
+ },
+ duration: {
+ label: i18n.translate('xpack.monitoring.alerts.memoryUsage.paramDetails.duration.label', {
+ defaultMessage: `Look at the average over`,
+ }),
+ type: AlertParamType.Duration,
+ },
+ },
+ label: i18n.translate('xpack.monitoring.alerts.memoryUsage.label', {
+ defaultMessage: 'Memory Usage (JVM)',
+ }),
+ },
+ [ALERT_MISSING_MONITORING_DATA]: {
+ paramDetails: {
+ duration: {
+ label: i18n.translate('xpack.monitoring.alerts.missingData.paramDetails.duration.label', {
+ defaultMessage: `Notify if monitoring data is missing for the last`,
+ }),
+ type: AlertParamType.Duration,
+ } as CommonAlertParamDetail,
+ limit: {
+ label: i18n.translate('xpack.monitoring.alerts.missingData.paramDetails.limit.label', {
+ defaultMessage: `looking back`,
+ }),
+ type: AlertParamType.Duration,
+ } as CommonAlertParamDetail,
+ },
+ label: i18n.translate('xpack.monitoring.alerts.missingData.label', {
+ defaultMessage: 'Missing monitoring data',
+ }),
+ },
+ [ALERT_THREAD_POOL_SEARCH_REJECTIONS]: {
+ paramDetails: {
+ threshold: {
+ label: i18n.translate('xpack.monitoring.alerts.rejection.paramDetails.threshold.label', {
+ defaultMessage: `Notify when {type} rejection count is over`,
+ values: { type: 'search' },
+ }),
+ type: AlertParamType.Number,
+ },
+ duration: {
+ label: i18n.translate('xpack.monitoring.alerts.rejection.paramDetails.duration.label', {
+ defaultMessage: `In the last`,
+ }),
+ type: AlertParamType.Duration,
+ },
+ },
+ label: i18n.translate('xpack.monitoring.alerts.threadPoolRejections.label', {
+ defaultMessage: 'Thread pool {type} rejections',
+ values: { type: 'search' },
+ }),
+ },
+ [ALERT_THREAD_POOL_WRITE_REJECTIONS]: {
+ paramDetails: {
+ threshold: {
+ label: i18n.translate('xpack.monitoring.alerts.rejection.paramDetails.threshold.label', {
+ defaultMessage: `Notify when {type} rejection count is over`,
+ values: { type: 'write' },
+ }),
+ type: AlertParamType.Number,
+ },
+ duration: {
+ label: i18n.translate('xpack.monitoring.alerts.rejection.paramDetails.duration.label', {
+ defaultMessage: `In the last`,
+ }),
+ type: AlertParamType.Duration,
+ },
+ },
+ label: i18n.translate('xpack.monitoring.alerts.threadPoolRejections.label', {
+ defaultMessage: 'Thread pool {type} rejections',
+ values: { type: 'write' },
+ }),
+ },
+};
 
 /**
  * A listing of all alert types
@@ -253,6 +410,8 @@ export const ALERTS = [
  ALERT_LOGSTASH_VERSION_MISMATCH,
  ALERT_MEMORY_USAGE,
  ALERT_MISSING_MONITORING_DATA,
+ ALERT_THREAD_POOL_SEARCH_REJECTIONS,
+ ALERT_THREAD_POOL_WRITE_REJECTIONS,
 ];
 
 /**

x-pack/plugins/monitoring/common/enums.ts

@@ -25,6 +25,7 @@ export enum AlertMessageTokenType {
 export enum AlertParamType {
  Duration = 'duration',
  Percentage = 'percentage',
+ Number = 'number',
 }
 
 export enum SetupModeFeature {

x-pack/plugins/monitoring/server/alerts/types.d.ts → x-pack/plugins/monitoring/common/types/alerts.ts

@@ -3,16 +3,70 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-import { AlertMessageTokenType, AlertSeverity } from '../../common/enums';
-import { AlertInstanceState as BaseAlertInstanceState } from '../../../alerts/server';
+
+import { Alert } from '../../../alerts/common';
+import { AlertParamType, AlertMessageTokenType, AlertSeverity } from '../enums';
+
+export interface CommonBaseAlert {
+ type: string;
+ label: string;
+ paramDetails: CommonAlertParamDetails;
+ rawAlert: Alert;
+ isLegacy: boolean;
+}
+
+export interface CommonAlertStatus {
+ exists: boolean;
+ enabled: boolean;
+ states: CommonAlertState[];
+ alert: CommonBaseAlert;
+}
+
+export interface CommonAlertState {
+ firing: boolean;
+ state: any;
+ meta: any;
+}
+
+export interface CommonAlertFilter {
+ nodeUuid?: string;
+}
+
+export interface CommonAlertNodeUuidFilter extends CommonAlertFilter {
+ nodeUuid: string;
+}
+
+export interface CommonAlertStackProductFilter extends CommonAlertFilter {
+ stackProduct: string;
+}
+
+export interface CommonAlertParamDetail {
+ label: string;
+ type?: AlertParamType;
+}
+
+export interface CommonAlertParamDetails {
+ [name: string]: CommonAlertParamDetail | undefined;
+}
+
+export interface CommonAlertParams {
+ [name: string]: string | number;
+}
+
+export interface ThreadPoolRejectionsAlertParams {
+ threshold: number;
+ duration: string;
+}
 
 export interface AlertEnableAction {
  id: string;
  config: { [key: string]: any };
 }
 
 export interface AlertInstanceState {
- alertStates: Array<AlertState | AlertCpuUsageState | AlertDiskUsageState>;
+ alertStates: Array<
+ AlertState | AlertCpuUsageState | AlertDiskUsageState | AlertThreadPoolRejectionsState
+ >;
  [x: string]: unknown;
 }
 
@@ -46,6 +100,13 @@ export interface AlertMemoryUsageState extends AlertNodeState {
  memoryUsage: number;
 }
 
+export interface AlertThreadPoolRejectionsState extends AlertState {
+ rejectionCount: number;
+ type: string;
+ nodeId: string;
+ nodeName?: string;
+}
+
 export interface AlertUiState {
  isFiring: boolean;
  severity: AlertSeverity;
@@ -100,6 +161,14 @@ export interface AlertCpuUsageNodeStats extends AlertNodeStats {
  containerQuota: number;
 }
 
+export interface AlertThreadPoolRejectionsStats {
+ clusterUuid: string;
+ nodeId: string;
+ nodeName: string;
+ rejectionCount: number;
+ ccs?: string;
+}
+
 export interface AlertDiskUsageNodeStats extends AlertNodeStats {
  diskUsage: number;
 }
@@ -121,7 +190,7 @@ export interface AlertData {
  instanceKey: string;
  clusterUuid: string;
  ccs?: string;
- shouldFire: boolean;
+ shouldFire?: boolean;
  severity: AlertSeverity;
  meta: any;
 }