Fix Filebeat tests with new region_iso_code field (elastic#7678)

In elastic/elasticsearch#31669 the field `region_iso_code` was added to the geoip processor. Because of this test broke with the most recent release of Elasticsearch as the events contain an undocumented field.
adriansr · Jul 23, 2018 · e2ddff7 · e2ddff7
1 parent b2fb25d
commit e2ddff7
Show file tree

Hide file tree

Showing 17 changed files with 123 additions and 18 deletions.
diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc
@@ -305,6 +305,16 @@ type: keyword
 The city name.
 
 
+--
+
+*`apache2.access.geoip.region_iso_code`*::
++
+--
+type: keyword
+
+Region ISO code.
+
+
 --
 
 [float]
@@ -556,6 +566,16 @@ type: geo_point
 The longitude and latitude.
 
 
+--
+
+*`auditd.log.geoip.region_iso_code`*::
++
+--
+type: keyword
+
+Region ISO code.
+
+
 --
 
 [[exported-fields-beat]]
@@ -2000,6 +2020,16 @@ type: keyword
 The city name.
 
 
+--
+
+*`iis.access.geoip.region_iso_code`*::
++
+--
+type: keyword
+
+Region ISO code.
+
+
 --
 
 [float]
@@ -2166,6 +2196,16 @@ type: keyword
 The city name.
 
 
+--
+
+*`iis.error.geoip.region_iso_code`*::
++
+--
+type: keyword
+
+Region ISO code.
+
+
 --
 
 [[exported-fields-kafka]]
@@ -3353,6 +3393,16 @@ type: keyword
 The city name.
 
 
+--
+
+*`nginx.access.geoip.region_iso_code`*::
++
+--
+type: keyword
+
+Region ISO code.
+
+
 --
 
 [float]
@@ -3890,6 +3940,16 @@ type: geo_point
 The longitude and latitude.
 
 
+--
+
+*`system.auth.ssh.geoip.region_iso_code`*::
++
+--
+type: keyword
+
+Region ISO code.
+
+
 --
 
 [float]
@@ -4333,6 +4393,16 @@ type: keyword
 The city name.
 
 
+--
+
+*`traefik.access.geoip.region_iso_code`*::
++
+--
+type: keyword
+
+Region ISO code.
+
+
 --
 
 *`traefik.access.request_count`*::

diff --git a/filebeat/include/fields.go b/filebeat/include/fields.go
diff --git a/filebeat/module/apache2/access/_meta/fields.yml b/filebeat/module/apache2/access/_meta/fields.yml
@@ -112,4 +112,7 @@
  type: keyword
  description: >
  The city name.
-
+ - name: region_iso_code
+ type: keyword
+ description: >
+ Region ISO code.
diff --git a/filebeat/module/auditd/log/_meta/fields.yml b/filebeat/module/auditd/log/_meta/fields.yml
@@ -77,3 +77,7 @@
  type: geo_point
  description: >
  The longitude and latitude.
+ - name: region_iso_code
+ type: keyword
+ description: >
+ Region ISO code.
diff --git a/filebeat/module/icinga/startup/test/test.log-expected.json b/filebeat/module/icinga/startup/test/test.log-expected.json
@@ -1,6 +1,6 @@
 [
  {
- "@timestamp": "2018-06-27T06:22:36.186Z", 
+ "@timestamp": "2018-07-23T11:50:38.896Z", 
  "fileset.module": "icinga", 
  "fileset.name": "startup", 
  "icinga.startup.facility": "cli", 
@@ -11,7 +11,7 @@
  "prospector.type": "log"
  }, 
  {
- "@timestamp": "2018-06-27T06:22:36.186Z", 
+ "@timestamp": "2018-07-23T11:50:38.896Z", 
  "fileset.module": "icinga", 
  "fileset.name": "startup", 
  "icinga.startup.facility": "cli", 

diff --git a/filebeat/module/iis/access/_meta/fields.yml b/filebeat/module/iis/access/_meta/fields.yml
@@ -157,3 +157,7 @@
  type: keyword
  description: >
  The city name.
+ - name: region_iso_code
+ type: keyword
+ description: >
+ Region ISO code.
diff --git a/filebeat/module/iis/access/test/test.log-expected.json b/filebeat/module/iis/access/test/test.log-expected.json
@@ -8,6 +8,7 @@
  "iis.access.geoip.country_iso_code": "DE", 
  "iis.access.geoip.location.lat": 52.5167, 
  "iis.access.geoip.location.lon": 13.4, 
+ "iis.access.geoip.region_iso_code": "DE-BE", 
  "iis.access.geoip.region_name": "Land Berlin", 
  "iis.access.method": "GET", 
  "iis.access.port": "80", 
@@ -73,6 +74,7 @@
  "iis.access.geoip.country_iso_code": "DE", 
  "iis.access.geoip.location.lat": 52.5167, 
  "iis.access.geoip.location.lon": 13.4, 
+ "iis.access.geoip.region_iso_code": "DE-BE", 
  "iis.access.geoip.region_name": "Land Berlin", 
  "iis.access.hostname": "example.com", 
  "iis.access.http_version": "1.1", 

diff --git a/filebeat/module/iis/error/_meta/fields.yml b/filebeat/module/iis/error/_meta/fields.yml
@@ -71,3 +71,7 @@
  type: keyword
  description: >
  The city name.
+ - name: region_iso_code
+ type: keyword
+ description: >
+ Region ISO code.
diff --git a/filebeat/module/iis/error/test/test.log-expected.json b/filebeat/module/iis/error/test/test.log-expected.json
@@ -26,6 +26,7 @@
  "iis.error.geoip.country_iso_code": "DE", 
  "iis.error.geoip.location.lat": 52.5167, 
  "iis.error.geoip.location.lon": 13.4, 
+ "iis.error.geoip.region_iso_code": "DE-BE", 
  "iis.error.geoip.region_name": "Land Berlin", 
  "iis.error.http_version": "1.1", 
  "iis.error.method": "GET", 
@@ -50,6 +51,7 @@
  "iis.error.geoip.country_iso_code": "DE", 
  "iis.error.geoip.location.lat": 52.5167, 
  "iis.error.geoip.location.lon": 13.4, 
+ "iis.error.geoip.region_iso_code": "DE-BE", 
  "iis.error.geoip.region_name": "Land Berlin", 
  "iis.error.http_version": "2.0", 
  "iis.error.method": "GET", 
@@ -74,6 +76,7 @@
  "iis.error.geoip.country_iso_code": "DE", 
  "iis.error.geoip.location.lat": 52.5167, 
  "iis.error.geoip.location.lon": 13.4, 
+ "iis.error.geoip.region_iso_code": "DE-BE", 
  "iis.error.geoip.region_name": "Land Berlin", 
  "iis.error.queue_name": "-", 
  "iis.error.reason_phrase": "Timer_MinBytesPerSecond", 

diff --git a/filebeat/module/kafka/log/test/state-change-1.1.0.log-expected.json b/filebeat/module/kafka/log/test/state-change-1.1.0.log-expected.json
@@ -1,15 +1,15 @@
 [
  {
- "@timestamp": "2018-07-16T10:17:06.489Z",
- "fileset.name": "log",
- "fileset.module": "kafka",
- "kafka.log.message": "Cached leader info PartitionState(controllerEpoch=25, leader=-1, leaderEpoch=15, isr=[10], zkVersion=15, replicas=[10], offlineReplicas=[10]) for partition __consumer_offsets-16 in response to UpdateMetadata request sent by controller 20 epoch 25 with correlation id 8",
- "kafka.log.component": "Broker id=30",
- "kafka.log.class": "state.change.logger",
- "kafka.log.level": "TRACE",
- "message": "[2018-07-16 10:17:06,489] TRACE [Broker id=30] Cached leader info PartitionState(controllerEpoch=25, leader=-1, leaderEpoch=15, isr=[10], zkVersion=15, replicas=[10], offlineReplicas=[10]) for partition __consumer_offsets-16 in response to UpdateMetadata request sent by controller 20 epoch 25 with correlation id 8 (state.change.logger)",
- "offset": 0,
- "input.type": "log",
+ "@timestamp": "2018-07-16T10:17:06.489Z", 
+ "fileset.module": "kafka", 
+ "fileset.name": "log", 
+ "input.type": "log", 
+ "kafka.log.class": "state.change.logger", 
+ "kafka.log.component": "Broker id=30", 
+ "kafka.log.level": "TRACE", 
+ "kafka.log.message": "Cached leader info PartitionState(controllerEpoch=25, leader=-1, leaderEpoch=15, isr=[10], zkVersion=15, replicas=[10], offlineReplicas=[10]) for partition __consumer_offsets-16 in response to UpdateMetadata request sent by controller 20 epoch 25 with correlation id 8", 
+ "message": "[2018-07-16 10:17:06,489] TRACE [Broker id=30] Cached leader info PartitionState(controllerEpoch=25, leader=-1, leaderEpoch=15, isr=[10], zkVersion=15, replicas=[10], offlineReplicas=[10]) for partition __consumer_offsets-16 in response to UpdateMetadata request sent by controller 20 epoch 25 with correlation id 8 (state.change.logger)", 
+ "offset": 0, 
  "prospector.type": "log"
  }
-]
+]
diff --git a/filebeat/module/nginx/access/_meta/fields.yml b/filebeat/module/nginx/access/_meta/fields.yml
@@ -119,4 +119,7 @@
  type: keyword
  description: >
  The city name.
-
+ - name: region_iso_code
+ type: keyword
+ description: >
+ Region ISO code.
diff --git a/filebeat/module/nginx/access/test/test.log-expected.json b/filebeat/module/nginx/access/test/test.log-expected.json
@@ -65,6 +65,7 @@
  "nginx.access.geoip.country_iso_code": "DE", 
  "nginx.access.geoip.location.lat": 52.5167, 
  "nginx.access.geoip.location.lon": 13.4, 
+ "nginx.access.geoip.region_iso_code": "DE-BE", 
  "nginx.access.geoip.region_name": "Land Berlin", 
  "nginx.access.http_version": "1.1", 
  "nginx.access.method": "GET", 
@@ -100,6 +101,7 @@
  "nginx.access.geoip.country_iso_code": "DE", 
  "nginx.access.geoip.location.lat": 52.5167, 
  "nginx.access.geoip.location.lon": 13.4, 
+ "nginx.access.geoip.region_iso_code": "DE-BE", 
  "nginx.access.geoip.region_name": "Land Berlin", 
  "nginx.access.http_version": "1.1", 
  "nginx.access.method": "GET", 
@@ -133,6 +135,7 @@
  "nginx.access.geoip.country_iso_code": "US", 
  "nginx.access.geoip.location.lat": 39.772, 
  "nginx.access.geoip.location.lon": -89.6859, 
+ "nginx.access.geoip.region_iso_code": "US-IL", 
  "nginx.access.geoip.region_name": "Illinois", 
  "nginx.access.http_version": "1.1", 
  "nginx.access.method": "GET", 

diff --git a/filebeat/module/system/auth/_meta/fields.yml b/filebeat/module/system/auth/_meta/fields.yml
@@ -79,7 +79,10 @@
  type: geo_point
  description: >
  The longitude and latitude.
-
+ - name: region_iso_code
+ type: keyword
+ description: >
+ Region ISO code.
  - name: sudo
  type: group
  description: >

diff --git a/filebeat/module/system/auth/test/test.log-expected.json b/filebeat/module/system/auth/test/test.log-expected.json
@@ -61,6 +61,7 @@
  "system.auth.ssh.geoip.country_iso_code": "CN", 
  "system.auth.ssh.geoip.location.lat": 22.5333, 
  "system.auth.ssh.geoip.location.lon": 114.1333, 
+ "system.auth.ssh.geoip.region_iso_code": "CN-44", 
  "system.auth.ssh.geoip.region_name": "Guangdong", 
  "system.auth.ssh.ip": "116.31.116.24", 
  "system.auth.ssh.method": "password", 

diff --git a/filebeat/module/traefik/access/_meta/fields.yml b/filebeat/module/traefik/access/_meta/fields.yml
@@ -112,6 +112,10 @@
  type: keyword
  description: >
  The city name.
+ - name: region_iso_code
+ type: keyword
+ description: >
+ Region ISO code.
  - name: request_count
  type: long
  description: >

diff --git a/filebeat/module/traefik/access/test/test.log-expected.json b/filebeat/module/traefik/access/test/test.log-expected.json
@@ -35,6 +35,7 @@
  "traefik.access.geoip.country_iso_code": "DE", 
  "traefik.access.geoip.location.lat": 52.5167, 
  "traefik.access.geoip.location.lon": 13.4, 
+ "traefik.access.geoip.region_iso_code": "DE-BE", 
  "traefik.access.geoip.region_name": "Land Berlin", 
  "traefik.access.http_version": "1.1", 
  "traefik.access.method": "GET", 

diff --git a/testing/environments/args.yml b/testing/environments/args.yml
@@ -7,4 +7,4 @@ services:
  args:
  DOWNLOAD_URL: https://snapshots.elastic.co/downloads
  ELASTIC_VERSION: 7.0.0-alpha1-SNAPSHOT
- CACHE_BUST: 20180501
+ CACHE_BUST: 20180723