diff --git a/filebeat/docs/modules/aws.asciidoc b/filebeat/docs/modules/aws.asciidoc index 32d3eab6c9b..a1652498c96 100644 --- a/filebeat/docs/modules/aws.asciidoc +++ b/filebeat/docs/modules/aws.asciidoc @@ -197,6 +197,29 @@ Required when using temporary security credentials. *`var.role_arn`*:: AWS IAM Role to assume. +[float] +=== config behaviour +Beware that in case both `var.queue_url` and `var.bucket_arn` are not set +instead of failing to start Filebeat with a config validation error, only the +specific fileset input will be stopped and a warning printed: +``` +2021-08-26T14:33:03.661-0600 WARN [aws-s3] awss3/config.go:54 neither queue_url nor bucket_arn were provided, input aws-s3 will stop +2021-08-26T14:33:10.668-0600 INFO [input.aws-s3] compat/compat.go:111 Input aws-s3 starting {"id": "29F3565F5B2A7070"} +2021-08-26T14:33:10.668-0600 INFO [input.aws-s3] compat/compat.go:124 Input 'aws-s3' stopped {"id": "29F3565F5B2A7070"} +``` + +This behaviour is required in order to reduce destruction of existing Filebeat setup +where not all AWS module's filesets are defined and will change in next major release. + +Setting `enabled: false` in the unused fileset will silence the warning and it is +the suggested setup. For example (assuming `cloudtrail` as unused fileset): +``` +- module: aws + cloudtrail: + enabled: false + +``` + [float] === cloudtrail fileset diff --git a/x-pack/filebeat/input/awss3/config.go b/x-pack/filebeat/input/awss3/config.go index 4e887003477..404997ddc60 100644 --- a/x-pack/filebeat/input/awss3/config.go +++ b/x-pack/filebeat/input/awss3/config.go @@ -12,6 +12,7 @@ import ( "github.com/elastic/beats/v7/libbeat/common/cfgtype" "github.com/elastic/beats/v7/libbeat/common/match" + "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/reader/parser" "github.com/elastic/beats/v7/libbeat/reader/readfile" "github.com/elastic/beats/v7/libbeat/reader/readfile/encoding" @@ -50,7 +51,8 @@ func defaultConfig() config { func (c *config) Validate() error { if c.QueueURL == "" && c.BucketARN == "" { - return fmt.Errorf("queue_url or bucket_arn must provided") + logp.NewLogger(inputName).Warnf("neither queue_url nor bucket_arn were provided, input %s will stop", inputName) + return nil } if c.QueueURL != "" && c.BucketARN != "" { diff --git a/x-pack/filebeat/input/awss3/config_test.go b/x-pack/filebeat/input/awss3/config_test.go index 9fdf4c1dffb..cd75d4df19c 100644 --- a/x-pack/filebeat/input/awss3/config_test.go +++ b/x-pack/filebeat/input/awss3/config_test.go @@ -92,7 +92,7 @@ func TestConfig(t *testing.T) { }, }, "", - func(queueURL, s3Bucketr string) config { + func(queueURL, s3Bucket string) config { c := makeConfig(queueURL, "") regex := match.MustCompile("/CloudTrail/") c.FileSelectors = []fileSelectorConfig{ @@ -112,8 +112,10 @@ func TestConfig(t *testing.T) { "queue_url": "", "bucket_arn": "", }, - "queue_url or bucket_arn must provided", - nil, + "", + func(queueURL, s3Bucket string) config { + return makeConfig("", "") + }, }, { "error on both queueURL and s3Bucket", diff --git a/x-pack/filebeat/module/aws/_meta/docs.asciidoc b/x-pack/filebeat/module/aws/_meta/docs.asciidoc index a36b1bd599b..f852da55a41 100644 --- a/x-pack/filebeat/module/aws/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/aws/_meta/docs.asciidoc @@ -192,6 +192,29 @@ Required when using temporary security credentials. *`var.role_arn`*:: AWS IAM Role to assume. +[float] +=== config behaviour +Beware that in case both `var.queue_url` and `var.bucket_arn` are not set +instead of failing to start Filebeat with a config validation error, only the +specific fileset input will be stopped and a warning printed: +``` +2021-08-26T14:33:03.661-0600 WARN [aws-s3] awss3/config.go:54 neither queue_url nor bucket_arn were provided, input aws-s3 will stop +2021-08-26T14:33:10.668-0600 INFO [input.aws-s3] compat/compat.go:111 Input aws-s3 starting {"id": "29F3565F5B2A7070"} +2021-08-26T14:33:10.668-0600 INFO [input.aws-s3] compat/compat.go:124 Input 'aws-s3' stopped {"id": "29F3565F5B2A7070"} +``` + +This behaviour is required in order to reduce destruction of existing Filebeat setup +where not all AWS module's filesets are defined and will change in next major release. + +Setting `enabled: false` in the unused fileset will silence the warning and it is +the suggested setup. For example (assuming `cloudtrail` as unused fileset): +``` +- module: aws + cloudtrail: + enabled: false + +``` + [float] === cloudtrail fileset