warn log entry and no validation failure when both queue_url and buck… (

elastic#27612)

* warn log entry and no validation failure when both queue_url and bucket_arn are not provided

* improve documentation

filebeat/docs/modules/aws.asciidoc

@@ -197,6 +197,29 @@ Required when using temporary security credentials.
 *`var.role_arn`*::
 AWS IAM Role to assume.
 
+[float]
+=== config behaviour
+Beware that in case both `var.queue_url` and `var.bucket_arn` are not set
+instead of failing to start Filebeat with a config validation error, only the
+specific fileset input will be stopped and a warning printed:
+```
+2021-08-26T14:33:03.661-0600 WARN [aws-s3] awss3/config.go:54 neither queue_url nor bucket_arn were provided, input aws-s3 will stop
+2021-08-26T14:33:10.668-0600 INFO [input.aws-s3] compat/compat.go:111 Input aws-s3 starting {"id": "29F3565F5B2A7070"}
+2021-08-26T14:33:10.668-0600 INFO [input.aws-s3] compat/compat.go:124 Input 'aws-s3' stopped {"id": "29F3565F5B2A7070"}
+```
+
+This behaviour is required in order to reduce destruction of existing Filebeat setup
+where not all AWS module's filesets are defined and will change in next major release.
+
+Setting `enabled: false` in the unused fileset will silence the warning and it is
+the suggested setup. For example (assuming `cloudtrail` as unused fileset):
+```
+- module: aws
+ cloudtrail:
+ enabled: false
+
+```
+
 [float]
 === cloudtrail fileset
 

x-pack/filebeat/input/awss3/config.go

@@ -12,6 +12,7 @@ import (
 
  "github.com/elastic/beats/v7/libbeat/common/cfgtype"
  "github.com/elastic/beats/v7/libbeat/common/match"
+ "github.com/elastic/beats/v7/libbeat/logp"
  "github.com/elastic/beats/v7/libbeat/reader/parser"
  "github.com/elastic/beats/v7/libbeat/reader/readfile"
  "github.com/elastic/beats/v7/libbeat/reader/readfile/encoding"
@@ -50,7 +51,8 @@ func defaultConfig() config {
 
 func (c *config) Validate() error {
  if c.QueueURL == "" && c.BucketARN == "" {
- return fmt.Errorf("queue_url or bucket_arn must provided")
+ logp.NewLogger(inputName).Warnf("neither queue_url nor bucket_arn were provided, input %s will stop", inputName)
+ return nil
  }
 
  if c.QueueURL != "" && c.BucketARN != "" {

x-pack/filebeat/input/awss3/config_test.go

@@ -92,7 +92,7 @@ func TestConfig(t *testing.T) {
  },
  },
  "",
- func(queueURL, s3Bucketr string) config {
+ func(queueURL, s3Bucket string) config {
  c := makeConfig(queueURL, "")
  regex := match.MustCompile("/CloudTrail/")
  c.FileSelectors = []fileSelectorConfig{
@@ -112,8 +112,10 @@ func TestConfig(t *testing.T) {
  "queue_url": "",
  "bucket_arn": "",
  },
- "queue_url or bucket_arn must provided",
- nil,
+ "",
+ func(queueURL, s3Bucket string) config {
+ return makeConfig("", "")
+ },
  },
  {
  "error on both queueURL and s3Bucket",

x-pack/filebeat/module/aws/_meta/docs.asciidoc

@@ -192,6 +192,29 @@ Required when using temporary security credentials.
 *`var.role_arn`*::
 AWS IAM Role to assume.
 
+[float]
+=== config behaviour
+Beware that in case both `var.queue_url` and `var.bucket_arn` are not set
+instead of failing to start Filebeat with a config validation error, only the
+specific fileset input will be stopped and a warning printed:
+```
+2021-08-26T14:33:03.661-0600 WARN [aws-s3] awss3/config.go:54 neither queue_url nor bucket_arn were provided, input aws-s3 will stop
+2021-08-26T14:33:10.668-0600 INFO [input.aws-s3] compat/compat.go:111 Input aws-s3 starting {"id": "29F3565F5B2A7070"}
+2021-08-26T14:33:10.668-0600 INFO [input.aws-s3] compat/compat.go:124 Input 'aws-s3' stopped {"id": "29F3565F5B2A7070"}
+```
+
+This behaviour is required in order to reduce destruction of existing Filebeat setup
+where not all AWS module's filesets are defined and will change in next major release.
+
+Setting `enabled: false` in the unused fileset will silence the warning and it is
+the suggested setup. For example (assuming `cloudtrail` as unused fileset):
+```
+- module: aws
+ cloudtrail:
+ enabled: false
+
+```
+
 [float]
 === cloudtrail fileset