x-pack/filebeat/input/gcs: allow absent credentials when using GCS wi…

…th ADC
elastic · Jul 2, 2024 · 36101d9 · 36101d9
1 parent 53bf67c
commit 36101d9
Show file tree

Hide file tree

Showing 3 changed files with 58 additions and 23 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -293,6 +293,7 @@ https:/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Add ability to remove request trace logs from http_endpoint input. {pull}40005[40005]
 - Add ability to remove request trace logs from entityanalytics input. {pull}40004[40004]
 - Relax constraint on Base DN in entity analytics Active Directory provider. {pull}40054[40054]
+- Allow absent credentials when using GCS with Application Default Credentials. {issue}39977[39977] {pull}40072[40072]
 
 *Auditbeat*
 

diff --git a/x-pack/filebeat/input/gcs/config.go b/x-pack/filebeat/input/gcs/config.go
@@ -5,8 +5,16 @@
 package gcs
 
 import (
+ "context"
+ "errors"
+ "fmt"
+ "io/fs"
+ "os"
  "time"
 
+ "cloud.google.com/go/pubsub"
+ "golang.org/x/oauth2/google"
+
  "github.com/elastic/beats/v7/libbeat/common/match"
 )
 
@@ -17,7 +25,7 @@ type config struct {
  // ProjectId - Defines the project id of the concerned gcs bucket in Google Cloud.
  ProjectId string `config:"project_id" validate:"required"`
  // Auth - Defines the authentication mechanism to be used for accessing the gcs bucket.
- Auth authConfig `config:"auth" validate:"required"`
+ Auth authConfig `config:"auth"`
  // MaxWorkers - Defines the maximum number of go routines that will be spawned.
  MaxWorkers *int `config:"max_workers,omitempty" validate:"max=5000"`
  // Poll - Defines if polling should be performed on the input bucket source.
@@ -71,3 +79,29 @@ type fileCredentialsConfig struct {
 type jsonCredentialsConfig struct {
  AccountKey string `config:"account_key"`
 }
+
+func (c authConfig) Validate() error {
+ // credentials_file
+ if c.CredentialsFile != nil {
+ _, err := os.Stat(c.CredentialsFile.Path)
+ if errors.Is(err, fs.ErrNotExist) {
+ return fmt.Errorf("credentials_file is configured, but the file %q cannot be found", c.CredentialsFile.Path)
+ } else {
+ return nil
+ }
+ }
+
+ // credentials_json
+ if c.CredentialsJSON != nil && len(c.CredentialsJSON.AccountKey) > 0 {
+ return nil
+ }
+
+ // Application Default Credentials (ADC)
+ _, err := google.FindDefaultCredentials(context.Background(), pubsub.ScopePubSub)
+ if err == nil {
+ return nil
+ }
+
+ return fmt.Errorf("no authentication credentials were configured or detected " +
+ "(credentials_file, credentials_json, and application default credentials (ADC))")
+}
diff --git a/x-pack/filebeat/input/gcs/input_test.go b/x-pack/filebeat/input/gcs/input_test.go
@@ -49,7 +49,7 @@ func Test_StorageClient(t *testing.T) {
  name: "SingleBucketWithPoll_NoErr",
  baseConfig: map[string]interface{}{
  "project_id": "elastic-sa",
- "auth.credentials_file.path": "/gcs_creds.json",
+ "auth.credentials_file.path": "testdata/gcs_creds.json",
  "max_workers": 2,
  "poll": true,
  "poll_interval": "5s",
@@ -70,7 +70,7 @@ func Test_StorageClient(t *testing.T) {
  name: "SingleBucketWithoutPoll_NoErr",
  baseConfig: map[string]interface{}{
  "project_id": "elastic-sa",
- "auth.credentials_file.path": "/gcs_creds.json",
+ "auth.credentials_file.path": "testdata/gcs_creds.json",
  "max_workers": 2,
  "poll": false,
  "poll_interval": "10s",
@@ -91,7 +91,7 @@ func Test_StorageClient(t *testing.T) {
  name: "TwoBucketsWithPoll_NoErr",
  baseConfig: map[string]interface{}{
  "project_id": "elastic-sa",
- "auth.credentials_file.path": "/gcs_creds.json",
+ "auth.credentials_file.path": "testdata/gcs_creds.json",
  "max_workers": 2,
  "poll": true,
  "poll_interval": "5s",
@@ -117,7 +117,7 @@ func Test_StorageClient(t *testing.T) {
  name: "TwoBucketsWithoutPoll_NoErr",
  baseConfig: map[string]interface{}{
  "project_id": "elastic-sa",
- "auth.credentials_file.path": "/gcs_creds.json",
+ "auth.credentials_file.path": "testdata/gcs_creds.json",
  "max_workers": 2,
  "poll": false,
  "poll_interval": "10s",
@@ -143,7 +143,7 @@ func Test_StorageClient(t *testing.T) {
  name: "SingleBucketWithPoll_InvalidBucketErr",
  baseConfig: map[string]interface{}{
  "project_id": "elastic-sa",
- "auth.credentials_file.path": "/gcs_creds.json",
+ "auth.credentials_file.path": "testdata/gcs_creds.json",
  "max_workers": 2,
  "poll": true,
  "poll_interval": "5s",
@@ -161,7 +161,7 @@ func Test_StorageClient(t *testing.T) {
  name: "SingleBucketWithoutPoll_InvalidBucketErr",
  baseConfig: map[string]interface{}{
  "project_id": "elastic-sa",
- "auth.credentials_file.path": "/gcs_creds.json",
+ "auth.credentials_file.path": "testdata/gcs_creds.json",
  "max_workers": 2,
  "poll": false,
  "poll_interval": "5s",
@@ -179,7 +179,7 @@ func Test_StorageClient(t *testing.T) {
  name: "TwoBucketsWithPoll_InvalidBucketErr",
  baseConfig: map[string]interface{}{
  "project_id": "elastic-sa",
- "auth.credentials_file.path": "/gcs_creds.json",
+ "auth.credentials_file.path": "testdata/gcs_creds.json",
  "max_workers": 2,
  "poll": true,
  "poll_interval": "5s",
@@ -200,7 +200,7 @@ func Test_StorageClient(t *testing.T) {
  name: "SingleBucketWithPoll_InvalidConfigValue",
  baseConfig: map[string]interface{}{
  "project_id": "elastic-sa",
- "auth.credentials_file.path": "/gcs_creds.json",
+ "auth.credentials_file.path": "testdata/gcs_creds.json",
  "max_workers": 5100,
  "poll": true,
  "poll_interval": "5s",
@@ -218,7 +218,7 @@ func Test_StorageClient(t *testing.T) {
  name: "TwoBucketWithPoll_InvalidConfigValue",
  baseConfig: map[string]interface{}{
  "project_id": "elastic-sa",
- "auth.credentials_file.path": "/gcs_creds.json",
+ "auth.credentials_file.path": "testdata/gcs_creds.json",
  "max_workers": 5100,
  "poll": true,
  "poll_interval": "5s",
@@ -239,7 +239,7 @@ func Test_StorageClient(t *testing.T) {
  name: "SingleBucketWithPoll_parseJSON",
  baseConfig: map[string]interface{}{
  "project_id": "elastic-sa",
- "auth.credentials_file.path": "/gcs_creds.json",
+ "auth.credentials_file.path": "testdata/gcs_creds.json",
  "max_workers": 1,
  "poll": true,
  "poll_interval": "5s",
@@ -261,7 +261,7 @@ func Test_StorageClient(t *testing.T) {
  name: "ReadJSON",
  baseConfig: map[string]interface{}{
  "project_id": "elastic-sa",
- "auth.credentials_file.path": "/gcs_creds.json",
+ "auth.credentials_file.path": "testdata/gcs_creds.json",
  "max_workers": 1,
  "poll": true,
  "poll_interval": "5s",
@@ -282,7 +282,7 @@ func Test_StorageClient(t *testing.T) {
  name: "ReadOctetStreamJSON",
  baseConfig: map[string]interface{}{
  "project_id": "elastic-sa",
- "auth.credentials_file.path": "/gcs_creds.json",
+ "auth.credentials_file.path": "testdata/gcs_creds.json",
  "max_workers": 1,
  "poll": true,
  "poll_interval": "5s",
@@ -302,7 +302,7 @@ func Test_StorageClient(t *testing.T) {
  name: "ReadNDJSON",
  baseConfig: map[string]interface{}{
  "project_id": "elastic-sa",
- "auth.credentials_file.path": "/gcs_creds.json",
+ "auth.credentials_file.path": "testdata/gcs_creds.json",
  "max_workers": 1,
  "poll": true,
  "poll_interval": "5s",
@@ -322,7 +322,7 @@ func Test_StorageClient(t *testing.T) {
  name: "ReadMultilineGzJSON",
  baseConfig: map[string]interface{}{
  "project_id": "elastic-sa",
- "auth.credentials_file.path": "/gcs_creds.json",
+ "auth.credentials_file.path": "testdata/gcs_creds.json",
  "max_workers": 1,
  "poll": true,
  "poll_interval": "5s",
@@ -342,7 +342,7 @@ func Test_StorageClient(t *testing.T) {
  name: "ReadJSONWithRootAsArray",
  baseConfig: map[string]interface{}{
  "project_id": "elastic-sa",
- "auth.credentials_file.path": "/gcs_creds.json",
+ "auth.credentials_file.path": "testdata/gcs_creds.json",
  "max_workers": 1,
  "poll": true,
  "poll_interval": "5s",
@@ -364,7 +364,7 @@ func Test_StorageClient(t *testing.T) {
  name: "FilterByTimeStampEpoch",
  baseConfig: map[string]interface{}{
  "project_id": "elastic-sa",
- "auth.credentials_file.path": "/gcs_creds.json",
+ "auth.credentials_file.path": "testdata/gcs_creds.json",
  "max_workers": 1,
  "poll": true,
  "poll_interval": "5s",
@@ -385,7 +385,7 @@ func Test_StorageClient(t *testing.T) {
  name: "FilterByFileSelectorRegexSingle",
  baseConfig: map[string]interface{}{
  "project_id": "elastic-sa",
- "auth.credentials_file.path": "/gcs_creds.json",
+ "auth.credentials_file.path": "testdata/gcs_creds.json",
  "max_workers": 1,
  "poll": true,
  "poll_interval": "5s",
@@ -409,7 +409,7 @@ func Test_StorageClient(t *testing.T) {
  name: "FilterByFileSelectorRegexMulti",
  baseConfig: map[string]interface{}{
  "project_id": "elastic-sa",
- "auth.credentials_file.path": "/gcs_creds.json",
+ "auth.credentials_file.path": "testdata/gcs_creds.json",
  "max_workers": 1,
  "poll": true,
  "poll_interval": "5s",
@@ -437,7 +437,7 @@ func Test_StorageClient(t *testing.T) {
  name: "ExpandEventListFromField",
  baseConfig: map[string]interface{}{
  "project_id": "elastic-sa",
- "auth.credentials_file.path": "/gcs_creds.json",
+ "auth.credentials_file.path": "testdata/gcs_creds.json",
  "max_workers": 1,
  "poll": true,
  "poll_interval": "5s",
@@ -463,7 +463,7 @@ func Test_StorageClient(t *testing.T) {
  name: "MultiContainerWithMultiFileSelectors",
  baseConfig: map[string]interface{}{
  "project_id": "elastic-sa",
- "auth.credentials_file.path": "/gcs_creds.json",
+ "auth.credentials_file.path": "testdata/gcs_creds.json",
  "max_workers": 1,
  "poll": true,
  "poll_interval": "5s",
@@ -496,7 +496,7 @@ func Test_StorageClient(t *testing.T) {
  name: "FilterByFileSelectorEmptyRegex",
  baseConfig: map[string]interface{}{
  "project_id": "elastic-sa",
- "auth.credentials_file.path": "/gcs_creds.json",
+ "auth.credentials_file.path": "testdata/gcs_creds.json",
  "max_workers": 1,
  "poll": true,
  "poll_interval": "5s",
@@ -536,7 +536,7 @@ func Test_StorageClient(t *testing.T) {
  conf := config{}
  err := cfg.Unpack(&conf)
  if err != nil {
- assert.EqualError(t, err, tt.isError.Error())
+ assert.EqualError(t, err, fmt.Sprint(tt.isError))
  return
  }
  input := newStatelessInput(conf)