From 40e86b7cc4ff320c6472cb141d5d7f045c52ba69 Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Tue, 27 Oct 2020 16:23:42 -0500 Subject: [PATCH 1/4] remove event.original from experimental schema --- experimental/generated/beats/fields.ecs.yml | 2 +- experimental/generated/csv/fields.csv | 2 +- experimental/generated/ecs/ecs_flat.yml | 2 +- experimental/generated/ecs/ecs_nested.yml | 2 +- experimental/generated/elasticsearch/7/template.json | 2 -- experimental/schemas/event.yml | 5 ----- rfcs/text/0001/event.yml | 1 + 7 files changed, 5 insertions(+), 11 deletions(-) delete mode 100644 experimental/schemas/event.yml diff --git a/experimental/generated/beats/fields.ecs.yml b/experimental/generated/beats/fields.ecs.yml index be3a96763c..9e725e93f9 100644 --- a/experimental/generated/beats/fields.ecs.yml +++ b/experimental/generated/beats/fields.ecs.yml @@ -1324,7 +1324,7 @@ but it can be retrieved from `_source`.' example: Sep 19 08:26:10 host CEF:0|Security| threatmanager|1.0|100| worm successfully stopped|10|src=10.0.0.1 dst=2.1.2.2spt=1232 - index: false + index: true - name: outcome level: core type: keyword diff --git a/experimental/generated/csv/fields.csv b/experimental/generated/csv/fields.csv index 964fa9acc2..7cc5d29b36 100644 --- a/experimental/generated/csv/fields.csv +++ b/experimental/generated/csv/fields.csv @@ -149,7 +149,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 2.0.0-dev,true,event,event.ingested,date,core,,2016-05-23T08:05:35.101Z,Timestamp when an event arrived in the central data store. 2.0.0-dev,true,event,event.kind,keyword,core,,alert,The kind of the event. The highest categorization field in the hierarchy. 2.0.0-dev,true,event,event.module,keyword,core,,apache,Name of the module this data is coming from. -2.0.0-dev,false,event,event.original,wildcard,core,,Sep 19 08:26:10 host CEF:0|Security| threatmanager|1.0|100| worm successfully stopped|10|src=10.0.0.1 dst=2.1.2.2spt=1232,Raw text message of entire event. +2.0.0-dev,true,event,event.original,wildcard,core,,Sep 19 08:26:10 host CEF:0|Security| threatmanager|1.0|100| worm successfully stopped|10|src=10.0.0.1 dst=2.1.2.2spt=1232,Raw text message of entire event. 2.0.0-dev,true,event,event.outcome,keyword,core,,success,The outcome of the event. The lowest level categorization field in the hierarchy. 2.0.0-dev,true,event,event.provider,keyword,extended,,kernel,Source of the event. 2.0.0-dev,true,event,event.reason,keyword,extended,,Terminated an unexpected process,"Reason why this event happened, according to the source" diff --git a/experimental/generated/ecs/ecs_flat.yml b/experimental/generated/ecs/ecs_flat.yml index b07d2ba201..efdda99ecf 100644 --- a/experimental/generated/ecs/ecs_flat.yml +++ b/experimental/generated/ecs/ecs_flat.yml @@ -2038,7 +2038,7 @@ event.original: example: Sep 19 08:26:10 host CEF:0|Security| threatmanager|1.0|100| worm successfully stopped|10|src=10.0.0.1 dst=2.1.2.2spt=1232 flat_name: event.original - index: false + index: true level: core name: original normalize: [] diff --git a/experimental/generated/ecs/ecs_nested.yml b/experimental/generated/ecs/ecs_nested.yml index ebd19083ed..89ad73bf49 100644 --- a/experimental/generated/ecs/ecs_nested.yml +++ b/experimental/generated/ecs/ecs_nested.yml @@ -2436,7 +2436,7 @@ event: example: Sep 19 08:26:10 host CEF:0|Security| threatmanager|1.0|100| worm successfully stopped|10|src=10.0.0.1 dst=2.1.2.2spt=1232 flat_name: event.original - index: false + index: true level: core name: original normalize: [] diff --git a/experimental/generated/elasticsearch/7/template.json b/experimental/generated/elasticsearch/7/template.json index c4cded14d5..5fac6ebcca 100644 --- a/experimental/generated/elasticsearch/7/template.json +++ b/experimental/generated/elasticsearch/7/template.json @@ -705,8 +705,6 @@ "type": "keyword" }, "original": { - "doc_values": false, - "index": false, "type": "wildcard" }, "outcome": { diff --git a/experimental/schemas/event.yml b/experimental/schemas/event.yml deleted file mode 100644 index 07daa3ac87..0000000000 --- a/experimental/schemas/event.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -- name: event - fields: - - name: original - type: wildcard diff --git a/rfcs/text/0001/event.yml b/rfcs/text/0001/event.yml index 07daa3ac87..0b50d6f942 100644 --- a/rfcs/text/0001/event.yml +++ b/rfcs/text/0001/event.yml @@ -2,4 +2,5 @@ - name: event fields: - name: original + index: true type: wildcard From be6db0e9268af9e80eabd9b6b303cf8213dd0472 Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Tue, 27 Oct 2020 16:44:10 -0500 Subject: [PATCH 2/4] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2150e1c38f..bece4f080f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ All notable changes to this project will be documented in this file based on the #### Bugfixes * Addressed issue where foreign reuses weren't using the user-supplied `as` value for their destination. #960 +* Experimental artifacts failed to install due to `event.original` index setting. #1053 #### Added From 0bfa75481b587882e228ef539350796622179954 Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Tue, 10 Nov 2020 11:46:45 -0600 Subject: [PATCH 3/4] generate experimental artifacts --- experimental/generated/beats/fields.ecs.yml | 5 +++-- experimental/generated/csv/fields.csv | 2 +- experimental/generated/ecs/ecs_flat.yml | 5 +++-- experimental/generated/ecs/ecs_nested.yml | 5 +++-- experimental/generated/elasticsearch/7/template.json | 5 ++++- 5 files changed, 14 insertions(+), 8 deletions(-) diff --git a/experimental/generated/beats/fields.ecs.yml b/experimental/generated/beats/fields.ecs.yml index 9e725e93f9..3a869a2f4b 100644 --- a/experimental/generated/beats/fields.ecs.yml +++ b/experimental/generated/beats/fields.ecs.yml @@ -1317,14 +1317,15 @@ example: apache - name: original level: core - type: wildcard + type: keyword + ignore_above: 1024 description: 'Raw text message of entire event. Used to demonstrate log integrity. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`.' example: Sep 19 08:26:10 host CEF:0|Security| threatmanager|1.0|100| worm successfully stopped|10|src=10.0.0.1 dst=2.1.2.2spt=1232 - index: true + index: false - name: outcome level: core type: keyword diff --git a/experimental/generated/csv/fields.csv b/experimental/generated/csv/fields.csv index 7cc5d29b36..f1fd5c1fab 100644 --- a/experimental/generated/csv/fields.csv +++ b/experimental/generated/csv/fields.csv @@ -149,7 +149,7 @@ ECS_Version,Indexed,Field_Set,Field,Type,Level,Normalization,Example,Description 2.0.0-dev,true,event,event.ingested,date,core,,2016-05-23T08:05:35.101Z,Timestamp when an event arrived in the central data store. 2.0.0-dev,true,event,event.kind,keyword,core,,alert,The kind of the event. The highest categorization field in the hierarchy. 2.0.0-dev,true,event,event.module,keyword,core,,apache,Name of the module this data is coming from. -2.0.0-dev,true,event,event.original,wildcard,core,,Sep 19 08:26:10 host CEF:0|Security| threatmanager|1.0|100| worm successfully stopped|10|src=10.0.0.1 dst=2.1.2.2spt=1232,Raw text message of entire event. +2.0.0-dev,false,event,event.original,keyword,core,,Sep 19 08:26:10 host CEF:0|Security| threatmanager|1.0|100| worm successfully stopped|10|src=10.0.0.1 dst=2.1.2.2spt=1232,Raw text message of entire event. 2.0.0-dev,true,event,event.outcome,keyword,core,,success,The outcome of the event. The lowest level categorization field in the hierarchy. 2.0.0-dev,true,event,event.provider,keyword,extended,,kernel,Source of the event. 2.0.0-dev,true,event,event.reason,keyword,extended,,Terminated an unexpected process,"Reason why this event happened, according to the source" diff --git a/experimental/generated/ecs/ecs_flat.yml b/experimental/generated/ecs/ecs_flat.yml index efdda99ecf..85fbad3e10 100644 --- a/experimental/generated/ecs/ecs_flat.yml +++ b/experimental/generated/ecs/ecs_flat.yml @@ -2038,12 +2038,13 @@ event.original: example: Sep 19 08:26:10 host CEF:0|Security| threatmanager|1.0|100| worm successfully stopped|10|src=10.0.0.1 dst=2.1.2.2spt=1232 flat_name: event.original - index: true + ignore_above: 1024 + index: false level: core name: original normalize: [] short: Raw text message of entire event. - type: wildcard + type: keyword event.outcome: allowed_values: - description: Indicates that this event describes a failed result. A common example diff --git a/experimental/generated/ecs/ecs_nested.yml b/experimental/generated/ecs/ecs_nested.yml index 89ad73bf49..1c6533c1a9 100644 --- a/experimental/generated/ecs/ecs_nested.yml +++ b/experimental/generated/ecs/ecs_nested.yml @@ -2436,12 +2436,13 @@ event: example: Sep 19 08:26:10 host CEF:0|Security| threatmanager|1.0|100| worm successfully stopped|10|src=10.0.0.1 dst=2.1.2.2spt=1232 flat_name: event.original - index: true + ignore_above: 1024 + index: false level: core name: original normalize: [] short: Raw text message of entire event. - type: wildcard + type: keyword event.outcome: allowed_values: - description: Indicates that this event describes a failed result. A common diff --git a/experimental/generated/elasticsearch/7/template.json b/experimental/generated/elasticsearch/7/template.json index 5fac6ebcca..2aa85eafa2 100644 --- a/experimental/generated/elasticsearch/7/template.json +++ b/experimental/generated/elasticsearch/7/template.json @@ -705,7 +705,10 @@ "type": "keyword" }, "original": { - "type": "wildcard" + "doc_values": false, + "ignore_above": 1024, + "index": false, + "type": "keyword" }, "outcome": { "ignore_above": 1024, From 2579e5e7dbef5bc20df0479c67056d3f86a8ab68 Mon Sep 17 00:00:00 2001 From: Eric Beahan Date: Tue, 10 Nov 2020 11:50:13 -0600 Subject: [PATCH 4/4] revert index setting --- rfcs/text/0001/event.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/rfcs/text/0001/event.yml b/rfcs/text/0001/event.yml index 0b50d6f942..07daa3ac87 100644 --- a/rfcs/text/0001/event.yml +++ b/rfcs/text/0001/event.yml @@ -2,5 +2,4 @@ - name: event fields: - name: original - index: true type: wildcard