[ML] Adds ML modules for Metrics UI Integration (#76460)

* adds metrics ml integration

* renames jobs, updates datafeeds

* adds allow_no_indices: true for datafeeds

* updates module ids in manifest

* adds custom urls

* adds module and individual job descriptions

* removes model plots

* updates terms agg sizes

* updates chunking config

* removes query and default index pattern from manifest, updates descriptions

Co-authored-by: Elastic Machine <[email protected]>

x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/logo.json

@@ -0,0 +1,3 @@
+{
+ "icon": "metricsApp"
+}

x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/manifest.json

@@ -0,0 +1,38 @@
+{
+ "id": "metrics_ui_hosts",
+ "title": "Metrics Hosts",
+ "description": "Detect anomalous memory and network behavior on hosts.",
+ "type": "Metricbeat Data",
+ "logoFile": "logo.json",
+ "jobs": [
+ {
+ "id": "hosts_memory_usage",
+ "file": "hosts_memory_usage.json"
+ },
+ {
+ "id": "hosts_network_in",
+ "file": "hosts_network_in.json"
+ },
+ {
+ "id": "hosts_network_out",
+ "file": "hosts_network_out.json"
+ }
+ ],
+ "datafeeds": [
+ {
+ "id": "datafeed-hosts_memory_usage",
+ "file": "datafeed_hosts_memory_usage.json",
+ "job_id": "hosts_memory_usage"
+ },
+ {
+ "id": "datafeed-hosts_network_in",
+ "file": "datafeed_hosts_network_in.json",
+ "job_id": "hosts_network_in"
+ },
+ {
+ "id": "datafeed-hosts_network_out",
+ "file": "datafeed_hosts_network_out.json",
+ "job_id": "hosts_network_out"
+ }
+ ]
+}

x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_memory_usage.json

@@ -0,0 +1,16 @@
+{
+ "job_id": "JOB_ID",
+ "indices": [
+ "INDEX_PATTERN_NAME"
+ ],
+ "indices_options": {
+ "allow_no_indices": true
+ },
+ "query": {
+ "bool": {
+ "must": [
+ {"exists": {"field": "system.memory"}}
+ ]
+ }
+ }
+}

x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_network_in.json

@@ -0,0 +1,40 @@
+{
+ "job_id": "JOB_ID",
+ "indices": [
+ "INDEX_PATTERN_NAME"
+ ],
+ "indices_options": {
+ "allow_no_indices": true
+ },
+ "query": {
+ "bool": {
+ "must": [
+ {"exists": {"field": "system.network"}}
+ ]
+ }
+ },
+ "chunking_config": {
+ "mode": "manual",
+ "time_span": "900s"
+ },
+ "aggregations": {
+ "host.name": {"terms": {"field": "host.name", "size": 100},
+ "aggregations": {
+ "buckets": {
+ "date_histogram": {"field": "@timestamp","fixed_interval": "5m"},
+ "aggregations": {
+ "@timestamp": {"max": {"field": "@timestamp"}},
+ "bytes_in_max": {"max": {"field": "system.network.in.bytes"}},
+ "bytes_in_derivative": {"derivative": {"buckets_path": "bytes_in_max"}},
+ "positive_only":{
+ "bucket_script": {
+ "buckets_path": {"in_derivative": "bytes_in_derivative.value"},
+ "script": "params.in_derivative > 0.0 ? params.in_derivative : 0.0"
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+}

x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_network_out.json

@@ -0,0 +1,40 @@
+{
+ "job_id": "JOB_ID",
+ "indices": [
+ "INDEX_PATTERN_NAME"
+ ],
+ "indices_options": {
+ "allow_no_indices": true
+ },
+ "query": {
+ "bool": {
+ "must": [
+ {"exists": {"field": "system.network"}}
+ ]
+ }
+ },
+ "chunking_config": {
+ "mode": "manual",
+ "time_span": "900s"
+ },
+ "aggregations": {
+ "host.name": {"terms": {"field": "host.name", "size": 100},
+ "aggregations": {
+ "buckets": {
+ "date_histogram": {"field": "@timestamp","fixed_interval": "5m"},
+ "aggregations": {
+ "@timestamp": {"max": {"field": "@timestamp"}},
+ "bytes_out_max": {"max": {"field": "system.network.out.bytes"}},
+ "bytes_out_derivative": {"derivative": {"buckets_path": "bytes_out_max"}},
+ "positive_only":{
+ "bucket_script": {
+ "buckets_path": {"out_derivative": "bytes_out_derivative.value"},
+ "script": "params.out_derivative > 0.0 ? params.out_derivative : 0.0"
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+}

x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_memory_usage.json

@@ -0,0 +1,50 @@
+{
+ "job_type": "anomaly_detector",
+ "groups": [
+ "hosts",
+ "metrics"
+ ],
+ "description": "Metrics: Hosts - Identify unusual spikes in memory usage across hosts.",
+ "analysis_config": {
+ "bucket_span": "15m",
+ "detectors": [
+ {
+ "detector_description": "max('system.memory.actual.used.pct')",
+ "function": "max",
+ "field_name": "system.memory.actual.used.pct",
+ "custom_rules": [
+ {
+ "actions": [
+ "skip_result"
+ ],
+ "conditions": [
+ {
+ "applies_to": "actual",
+ "operator": "lt",
+ "value": 0.1
+ }
+ ]
+ }
+ ]
+ }
+ ],
+ "influencers": [
+ "host.name"
+ ]
+ },
+ "data_description": {
+ "time_field": "@timestamp"
+ },
+ "analysis_limits": {
+ "model_memory_limit": "64mb"
+ },
+ "custom_settings": {
+ "created_by": "ml-module-metrics-ui-hosts",
+ "custom_urls": [
+ {
+ "url_name": "Host Metrics",
+ "url_value": "metrics/detail/host/$host.name$?metricTime=(autoReload:!f,refreshInterval:5000,time:(from:%27$earliest$%27,interval:%3E%3D1m,to:%27$latest$%27))"
+ }
+ ]
+ }
+ }

x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_network_in.json

@@ -0,0 +1,37 @@
+{
+ "job_type": "anomaly_detector",
+ "description": "Metrics: Hosts - Identify unusual spikes in inbound traffic across hosts.",
+ "groups": [
+ "hosts",
+ "metrics"
+ ],
+ "analysis_config": {
+ "bucket_span": "15m",
+ "detectors": [
+ {
+ "detector_description": "max(bytes_in_derivative)",
+ "function": "max",
+ "field_name": "bytes_in_derivative"
+ }
+ ],
+ "influencers": [
+ "host.name"
+ ],
+ "summary_count_field_name": "doc_count"
+ },
+ "data_description": {
+ "time_field": "@timestamp"
+ },
+ "analysis_limits": {
+ "model_memory_limit": "32mb"
+ },
+ "custom_settings": {
+ "created_by": "ml-module-metrics-ui-hosts",
+ "custom_urls": [
+ {
+ "url_name": "Host Metrics",
+ "url_value": "metrics/detail/host/$host.name$?metricTime=(autoReload:!f,refreshInterval:5000,time:(from:%27$earliest$%27,interval:%3E%3D1m,to:%27$latest$%27))"
+ }
+ ]
+ }
+ }

x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_network_out.json

@@ -0,0 +1,37 @@
+{
+ "job_type": "anomaly_detector",
+ "description": "Metrics: Hosts - Identify unusual spikes in outbound traffic across hosts.",
+ "groups": [
+ "hosts",
+ "metrics"
+ ],
+ "analysis_config": {
+ "bucket_span": "15m",
+ "detectors": [
+ {
+ "detector_description": "max(bytes_out_derivative)",
+ "function": "max",
+ "field_name": "bytes_out_derivative"
+ }
+ ],
+ "influencers": [
+ "host.name"
+ ],
+ "summary_count_field_name": "doc_count"
+ },
+ "data_description": {
+ "time_field": "@timestamp"
+ },
+ "analysis_limits": {
+ "model_memory_limit": "32mb"
+ },
+ "custom_settings": {
+ "created_by": "ml-module-metrics-ui-hosts",
+ "custom_urls": [
+ {
+ "url_name": "Host Metrics",
+ "url_value": "metrics/detail/host/$host.name$?metricTime=(autoReload:!f,refreshInterval:5000,time:(from:%27$earliest$%27,interval:%3E%3D1m,to:%27$latest$%27))"
+ }
+ ]
+ }
+ }

x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/logo.json

@@ -0,0 +1,3 @@
+{
+ "icon": "metricsApp"
+}

x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/manifest.json

@@ -0,0 +1,38 @@
+{
+ "id": "metrics_ui_k8s",
+ "title": "Metrics Kubernetes",
+ "description": "Detect anomalous memory and network behavior on Kubernetes pods.",
+ "type": "Metricbeat Data",
+ "logoFile": "logo.json",
+ "jobs": [
+ {
+ "id": "k8s_memory_usage",
+ "file": "k8s_memory_usage.json"
+ },
+ {
+ "id": "k8s_network_in",
+ "file": "k8s_network_in.json"
+ },
+ {
+ "id": "k8s_network_out",
+ "file": "k8s_network_out.json"
+ }
+ ],
+ "datafeeds": [
+ {
+ "id": "datafeed-k8s_memory_usage",
+ "file": "datafeed_k8s_memory_usage.json",
+ "job_id": "k8s_memory_usage"
+ },
+ {
+ "id": "datafeed-k8s_network_in",
+ "file": "datafeed_k8s_network_in.json",
+ "job_id": "k8s_network_in"
+ },
+ {
+ "id": "datafeed-k8s_network_out",
+ "file": "datafeed_k8s_network_out.json",
+ "job_id": "k8s_network_out"
+ }
+ ]
+ }

x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_memory_usage.json

@@ -0,0 +1,17 @@
+{
+ "job_id": "JOB_ID",
+ "indices": [
+ "INDEX_PATTERN_NAME"
+ ],
+ "indices_options": {
+ "allow_no_indices": true
+ },
+ "query": {
+ "bool": {
+ "must": [
+ {"exists": {"field": "kubernetes.pod.uid"}},
+ {"exists": {"field": "kubernetes.pod.memory"}}
+ ]
+ }
+ }
+}

x-pack/plugins/ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_network_in.json

@@ -0,0 +1,44 @@
+{
+ "job_id": "JOB_ID",
+ "indices": [
+ "INDEX_PATTERN_NAME"
+ ],
+ "indices_options": {
+ "allow_no_indices": true
+ },
+ "query": {
+ "bool": {
+ "must": [
+ {"exists": {"field": "kubernetes.pod.network"}}
+ ]
+ }
+ },
+ "chunking_config": {
+ "mode": "manual",
+ "time_span": "900s"
+ },
+ "aggregations": {
+ "kubernetes.namespace": {"terms": {"field": "kubernetes.namespace", "size": 25},
+ "aggregations": {
+ "kubernetes.pod.uid": {"terms": {"field": "kubernetes.pod.uid", "size": 100},
+ "aggregations": {
+ "buckets": {
+ "date_histogram": {"field": "@timestamp","fixed_interval": "5m"},
+ "aggregations": {
+ "@timestamp": {"max": {"field": "@timestamp"}},
+ "bytes_in_max": {"max": {"field": "kubernetes.pod.network.rx.bytes"}},
+ "bytes_in_derivative": {"derivative": {"buckets_path": "bytes_in_max"}},
+ "positive_only":{
+ "bucket_script": {
+ "buckets_path": {"in_derivative": "bytes_in_derivative.value"},
+ "script": "params.in_derivative > 0.0 ? params.in_derivative : 0.0"
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+}