s3blob/blob: support additional endpoint query parameters

aws/aws.go

@@ -26,6 +26,7 @@ import (
  "github.com/aws/aws-sdk-go/aws"
  "github.com/aws/aws-sdk-go/aws/client"
  "github.com/aws/aws-sdk-go/aws/credentials"
+ "github.com/aws/aws-sdk-go/aws/endpoints"
  "github.com/aws/aws-sdk-go/aws/session"
  "github.com/google/wire"
 )
@@ -81,6 +82,8 @@ func (co ConfigOverrider) ClientConfig(serviceName string, cfgs ...*aws.Config)
 // - endpoint: The endpoint URL (hostname only or fully qualified URI); sets aws.Config.Endpoint.
 // - disableSSL: A value of "true" disables SSL when sending requests; sets aws.Config.DisableSSL.
 // - s3ForcePathStyle: A value of "true" forces the request to use path-style addressing; sets aws.Config.S3ForcePathStyle.
+// - dualstack: A value of "true" enables dual stack (IPv4 and IPv6) endpoints
+// - fips: A value of "true" enables the use of FIPS endpoints
 func ConfigFromURLParams(q url.Values) (*aws.Config, error) {
  var cfg aws.Config
  for param, values := range q {
@@ -102,6 +105,20 @@ func ConfigFromURLParams(q url.Values) (*aws.Config, error) {
  return nil, fmt.Errorf("invalid value for query parameter %q: %v", param, err)
  }
  cfg.S3ForcePathStyle = aws.Bool(b)
+ case "dualstack":
+ b, err := strconv.ParseBool(value)
+ if err != nil {
+ return nil, fmt.Errorf("invalid value for query parameter %q: %v", param, err)
+ }
+ cfg.UseDualStack = aws.Bool(b)
+ case "fips":
+ b, err := strconv.ParseBool(value)
+ if err != nil {
+ return nil, fmt.Errorf("invalid value for query parameter %q: %v", param, err)
+ }
+ if b {
+ cfg.UseFIPSEndpoint = endpoints.FIPSEndpointStateEnabled
+ }
  case "awssdk":
  // ignore, should be handled before this
  default:
@@ -178,6 +195,8 @@ func NewDefaultV2Config(ctx context.Context) (awsv2.Config, error) {
 // - profile: The shared config profile to use; sets SharedConfigProfile.
 // - endpoint: The AWS service endpoint to send HTTP request.
 // - hostname_immutable: Make the hostname immutable, only works if endpoint is also set.
+// - dualstack: A value of "true" enables dual stack (IPv4 and IPv6) endpoints.
+// - fips: A value of "true" enables the use of FIPS endpoints.
 func V2ConfigFromURLParams(ctx context.Context, q url.Values) (awsv2.Config, error) {
  var endpoint string
  var hostnameImmutable bool
@@ -197,6 +216,22 @@ func V2ConfigFromURLParams(ctx context.Context, q url.Values) (awsv2.Config, err
  endpoint = value
  case "profile":
  opts = append(opts, awsv2cfg.WithSharedConfigProfile(value))
+ case "dualstack":
+ dualStack, err := strconv.ParseBool(value)
+ if err != nil {
+ return awsv2.Config{}, fmt.Errorf("invalid value for dualstack: %w", err)
+ }
+ if dualStack {
+ opts = append(opts, awsv2cfg.WithUseDualStackEndpoint(awsv2.DualStackEndpointStateEnabled))
+ }
+ case "fips":
+ fips, err := strconv.ParseBool(value)
+ if err != nil {
+ return awsv2.Config{}, fmt.Errorf("invalid value for fips: %w", err)
+ }
+ if fips {
+ opts = append(opts, awsv2cfg.WithUseFIPSEndpoint(awsv2.FIPSEndpointStateEnabled))
+ }
  case "awssdk":
  // ignore, should be handled before this
  default:
@@ -215,5 +250,6 @@ func V2ConfigFromURLParams(ctx context.Context, q url.Values) (awsv2.Config, err
  })
  opts = append(opts, awsv2cfg.WithEndpointResolverWithOptions(customResolver))
  }
+
  return awsv2cfg.LoadDefaultConfig(ctx, opts...)
 }

aws/aws_test.go

@@ -184,6 +184,10 @@ func TestV2ConfigFromURLParams(t *testing.T) {
  HostnameImmutable: true,
  },
  },
+ {
+ name: "FIPS and dual stack",
+ query: url.Values{"fips": {"true"}, "dualstack": {"true"}},
+ },
  // Can't test "profile", since AWS validates that the profile exists.
  }
 

blob/s3blob/s3blob.go

@@ -130,6 +130,11 @@ const Scheme = "s3"
 // Use "awssdk=v1" to force using AWS SDK v1, "awssdk=v2" to force using AWS SDK v2,
 // or anything else to accept the default.
 //
+// The following S3-specific query options are also supported:
+// - ssetype: The type of server side encryption used (AES256, aws:kms, aws:kms:dsse)
+// - kmskeyid: The KMS key ID for server side encryption
+// - accelerate: A value of "true" uses the S3 Transfer Accleration endpoints
+//
 // For V1, see gocloud.dev/aws/ConfigFromURLParams for supported query parameters
 // for overriding the aws.Session from the URL.
 // For V2, see gocloud.dev/aws/V2ConfigFromURLParams.
@@ -145,8 +150,9 @@ type URLOpener struct {
 }
 
 const (
- sseTypeParamKey = "ssetype"
- kmsKeyIdParamKey = "kmskeyid"
+ sseTypeParamKey = "ssetype"
+ kmsKeyIdParamKey = "kmskeyid"
+ accelerateParamKey = "accelerate"
 )
 
 func toServerSideEncryptionType(value string) (typesv2.ServerSideEncryption, error) {
@@ -178,12 +184,24 @@ func (o *URLOpener) OpenBucketURL(ctx context.Context, u *url.URL) (*blob.Bucket
  o.Options.KMSEncryptionID = kmsKeyID
  }
 
+ accelerate := false
+ if accelerateParam := q.Get(accelerateParamKey); accelerateParam != "" {
+ q.Del(accelerateParamKey)
+ var err error
+ accelerate, err = strconv.ParseBool(accelerateParam)
+ if err != nil {
+ return nil, fmt.Errorf("invalid value for %q: %v", accelerateParamKey, err)
+ }
+ }
+
  if o.UseV2 {
  cfg, err := gcaws.V2ConfigFromURLParams(ctx, q)
  if err != nil {
  return nil, fmt.Errorf("open bucket %v: %v", u, err)
  }
- clientV2 := s3v2.NewFromConfig(cfg)
+ clientV2 := s3v2.NewFromConfig(cfg, func(o *s3v2.Options) {
+ o.UseAccelerate = accelerate
+ })
 
  return OpenBucketV2(ctx, clientV2, u.Host, &o.Options)
  }
@@ -194,6 +212,8 @@ func (o *URLOpener) OpenBucketURL(ctx context.Context, u *url.URL) (*blob.Bucket
  if err != nil {
  return nil, fmt.Errorf("open bucket %v: %v", u, err)
  }
+
+ overrideCfg.S3UseAccelerate = &accelerate
  configProvider.Configs = append(configProvider.Configs, overrideCfg)
 
  return OpenBucket(ctx, configProvider, u.Host, &o.Options)

blob/s3blob/s3blob_test.go

@@ -476,6 +476,26 @@ func TestOpenBucketFromURL(t *testing.T) {
  {"s3://mybucket?awssdk=v2", false},
  // OK, use KMS Server Side Encryption
  {"s3://mybucket?ssetype=aws:kms&kmskeyid=arn:aws:us-east-1:12345:key/1-a-2-b", false},
+ // OK, use S3 Transfer acceleration and dual stack endpoints
+ {"s3://mybucket?accelerate=true&dualstack=true", false},
+ // OK, use FIPS endpoints
+ {"s3://mybucket?fips=true", false},
+ // OK, use S3 Transfer accleration and dual stack endpoints (v1)
+ {"s3://mybucket?awssdk=v1&accelerate=true&dualstack=true", false},
+ // OK, use FIPS endpoints (v1)
+ {"s3://mybucket?awssdk=v1&fips=true", false},
+ // Invalid accelerate (v1)
+ {"s3://mybucket?awssdk=v1&accelerate=bogus", true},
+ // Invalid accelerate (v2)
+ {"s3://mybucket?accelerate=bogus", true},
+ // Invalid FIPS (v1)
+ {"s3://mybucket?awssdk=v1&fips=bogus", true},
+ // Invalid FIPS (v2)
+ {"s3://mybucket?fips=bogus", true},
+ // Invalid dualstack (v1)
+ {"s3://mybucket?awssdk=v1&dualstack=bad", true},
+ // Invalid dualstack (v2)
+ {"s3://mybucket?dualstack=bad", true},
  // Invalid ssetype
  {"s3://mybucket?ssetype=aws:notkmsoraes&kmskeyid=arn:aws:us-east-1:12345:key/1-a-2-b", true},
  // Invalid parameter together with a valid one.