add documentation

x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc

@@ -17,7 +17,7 @@ The use of SQS notification is preferred: polling list of S3 objects is expensiv
 in terms of performance and costs and should be preferably used only when no SQS
 notification can be attached to the S3 buckets. This input can, for example, be
 used to receive S3 access logs to monitor detailed records for the requests that
-are made to a bucket.
+are made to a bucket. This input also supports S3 notification from SNS to SQS.
 
 SQS notification method is enabled setting `queue_url` configuration value.
 S3 bucket list polling method is enabled setting `bucket_arn` configuration value.
@@ -386,6 +386,14 @@ create a notification through SQS. Please see
 https://docs.aws.amazon.com/AmazonS3/latest/dev/ways-to-add-notification-config-to-bucket.html#step1-create-sqs-queue-for-notification[create-sqs-queue-for-notification]
 for more details.
 
+[float]
+=== S3 -> SNS -> SQS setup
+If you would like to use the bucket notification in multiple different consumers
+(others than {beatname_lc}), you should use an SNS topic for the bucket notification.
+Please see https://docs.aws.amazon.com/AmazonS3/latest/userguide/ways-to-add-notification-config-to-bucket.html#step1-create-sns-topic-for-notification[create-SNS-topic-for-notification]
+for more details. SQS queue will be configured as a
+https://docs.aws.amazon.com/sns/latest/dg/sns-sqs-as-subscriber.html[subscriber to the SNS topic].
+
 [float]
 === Parallel Processing
 

x-pack/filebeat/input/awss3/input_integration_test.go

@@ -50,11 +50,11 @@ const (
 )
 
 type terraformOutputData struct {
- AWSRegion string `yaml:"aws_region"`
- BucketName string `yaml:"bucket_name"`
- QueueURL string `yaml:"queue_url"`
- BucketName2 string `yaml:"bucket_name_for_sns"`
- QueueURL2 string `yaml:"queue_url_for_sns"`
+ AWSRegion  string `yaml:"aws_region"`
+ BucketName  string `yaml:"bucket_name"`
+ QueueURL  string `yaml:"queue_url"`
+ BucketNameForSNS string `yaml:"bucket_name_for_sns"`
+ QueueURLForSNS string `yaml:"queue_url_for_sns"`
 }
 
 func getTerraformOutputs(t *testing.T) terraformOutputData {
@@ -458,12 +458,12 @@ func TestInputRunSNS(t *testing.T) {
  tfConfig := getTerraformOutputs(t)
 
  // Ensure SQS is empty before testing.
- drainSQS(t, tfConfig.AWSRegion, tfConfig.QueueURL2)
+ drainSQS(t, tfConfig.AWSRegion, tfConfig.QueueURLForSNS)
 
  // Ensure metrics are removed before testing.
  monitoring.GetNamespace("dataset").GetRegistry().Remove(inputID)
 
- uploadS3TestFiles(t, tfConfig.AWSRegion, tfConfig.BucketName2,
+ uploadS3TestFiles(t, tfConfig.AWSRegion, tfConfig.BucketNameForSNS,
  "testdata/events-array.json",
  "testdata/invalid.json",
  "testdata/log.json",
@@ -474,7 +474,7 @@ func TestInputRunSNS(t *testing.T) {
  "testdata/log.txt", // Skipped (no match).
  )
 
- s3Input := createInput(t, makeTestConfigSQS(tfConfig.QueueURL2))
+ s3Input := createInput(t, makeTestConfigSQS(tfConfig.QueueURLForSNS))
 
  inputCtx, cancel := newV2Context()
  t.Cleanup(cancel)
@@ -486,7 +486,6 @@ func TestInputRunSNS(t *testing.T) {
  defer close(client.Channel)
  go func() {
  for event := range client.Channel {
- // Fake the ACK handling that's not implemented in pubtest.
  event.Private.(*eventACKTracker).ACK()
  }
  }()