aws · mergify · Jul 31, 2023 · Mar 7, 2023 · Mar 7, 2023 · Mar 7, 2023
diff --git a/packages/@aws-cdk/aws-glue/README.md b/packages/@aws-cdk/aws-glue/README.md
@@ -225,7 +225,24 @@ new glue.Table(this, 'MyTable', {
 });
 ```
 
-By default, an S3 bucket will be created to store the table's data and stored in the bucket root. You can also manually pass the `bucket` and `s3Prefix`:
+Glue tables can be configured to contain user-defined properties, to describe the physical storage of table data, through the `storageParameters` property:
+
+```ts
+declare const myBucket: s3.Bucket;
+declare const myDatabase: glue.Database;
+new glue.Table(this, 'MyTable', {
+ database: myDatabase,
+ columns: [{
+ name: 'col1',
+ type: glue.Schema.STRING,
+ }],
+ dataFormat: glue.DataFormat.JSON,
+ storageParameters: {
+ 'skip.header.line.count': 1,
+ separatorChar: ',',
+ }
+});
+```
 
 ### Partition Keys
 

diff --git a/packages/@aws-cdk/aws-glue/lib/table.ts b/packages/@aws-cdk/aws-glue/lib/table.ts
@@ -183,6 +183,35 @@ export interface TableProps {
  * @default - The parameter is not defined
  */
  readonly enablePartitionFiltering?: boolean;
+
+ /**
+ * The user-supplied properties for the description of the physical storage of this table. These properties help describe the format of the data that is stored within the crawled data sources.
+ *
+ * There are reserved keys that are used by AWS Glue. They CAN be mutated, but they are best left alone.
+ *
+ * The key/value pairs that are allowed to be submitted are not limited, however their functionality is not guaranteed.
+ *
+ * @see https://docs.aws.amazon.com/redshift/latest/dg/r_CREATE_EXTERNAL_TABLE.html#r_CREATE_EXTERNAL_TABLE-parameters - under "TABLE PROPERTIES" contains a non-exhaustive list of the keys that have functionality.
+ *
+ * @example
+ *
+ * declare const glueDatabase: glue.IDatabase;
+ * const table = new glue.Table(this, 'Table', {
+ * database: glueDatabase,
+ * columns: [{
+ * name: 'col1',
+ * type: glue.Schema.STRING,
+ * }],
+ * dataFormat: glue.DataFormat.CSV,
+ * storageParameters: {
+ * foo: 'bar', // Will have no effect
+ * 'skip.header.line.count': 1, // Will be used to skip the first line of the file
+ * },
+ * });
+ *
+ * @default - The parameter is not defined
+ */
+ readonly storageParameters?: { [key: string]: any };
 }
 
 /**
@@ -330,6 +359,7 @@ export class Table extends Resource implements ITable {
  serdeInfo: {
  serializationLibrary: props.dataFormat.serializationLibrary.className,
  },
+ parameters: props.storageParameters,
  },
 
  tableType: 'EXTERNAL_TABLE',

diff --git a/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/aws-cdk-glue.assets.json b/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/aws-cdk-glue.assets.json
@@ -1,15 +1,15 @@
 {
- "version": "20.0.0",
+ "version": "30.1.0",
  "files": {
- "eef5abdc0f1ee16e5be447f60688757df6726f3c2d1d06c136e9bbdb99d96e1f": {
+ "553f4bc301289a3e09b1cd03c07892e4c62da6b35d057a57fe1613c230c27ef6": {
  "source": {
  "path": "aws-cdk-glue.template.json",
  "packaging": "file"
  },
  "destinations": {
  "current_account-current_region": {
  "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
- "objectKey": "eef5abdc0f1ee16e5be447f60688757df6726f3c2d1d06c136e9bbdb99d96e1f.json",
+ "objectKey": "553f4bc301289a3e09b1cd03c07892e4c62da6b35d057a57fe1613c230c27ef6.json",
  "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
  }
  }

diff --git a/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/aws-cdk-glue.template.json b/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/aws-cdk-glue.template.json
@@ -2,8 +2,8 @@
  "Resources": {
  "DataBucketE3889A50": {
  "Type": "AWS::S3::Bucket",
- "UpdateReplacePolicy": "Retain",
- "DeletionPolicy": "Retain"
+ "UpdateReplacePolicy": "Delete",
+ "DeletionPolicy": "Delete"
  },
  "MyDatabase1E2517DB": {
  "Type": "AWS::Glue::Database",
@@ -328,8 +328,8 @@
  "Version": "2012-10-17"
  }
  },
- "UpdateReplacePolicy": "Retain",
- "DeletionPolicy": "Retain"
+ "UpdateReplacePolicy": "Delete",
+ "DeletionPolicy": "Delete"
  },
  "MyEncryptedTableBucket7B28486D": {
  "Type": "AWS::S3::Bucket",
@@ -423,11 +423,6 @@
  }
  }
  },
- "MyPartitionFilteredTableBucket6ACAA137": {
- "Type": "AWS::S3::Bucket",
- "UpdateReplacePolicy": "Retain",
- "DeletionPolicy": "Retain"
- },
  "MyPartitionFilteredTable324BA27A": {
  "Type": "AWS::Glue::Table",
  "Properties": {
@@ -477,13 +472,18 @@
  [
  "s3://",
  {
- "Ref": "MyPartitionFilteredTableBucket6ACAA137"
+ "Ref": "DataBucketE3889A50"
  },
  "/"
  ]
  ]
  },
  "OutputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",
+ "Parameters": {
+ "separatorChar": ",",
+ "skip.header.line.count": 2,
+ "foo": "bar"
+ },
  "SerdeInfo": {
  "SerializationLibrary": "org.openx.data.jsonserde.JsonSerDe"
  },

diff --git a/...t/integ.table.js.snapshot/awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.assets.json b/...t/integ.table.js.snapshot/awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.assets.json
@@ -0,0 +1,19 @@
+{
+ "version": "30.1.0",
+ "files": {
+ "21fbb51d7b23f6a6c262b46a9caee79d744a3ac019fd45422d988b96d44b2a22": {
+ "source": {
+ "path": "awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.template.json",
+ "packaging": "file"
+ },
+ "destinations": {
+ "current_account-current_region": {
+ "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
+ "objectKey": "21fbb51d7b23f6a6c262b46a9caee79d744a3ac019fd45422d988b96d44b2a22.json",
+ "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
+ }
+ }
+ }
+ },
+ "dockerImages": {}
+}
diff --git a/...integ.table.js.snapshot/awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.template.json b/...integ.table.js.snapshot/awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.template.json
@@ -0,0 +1,36 @@
+{
+ "Parameters": {
+ "BootstrapVersion": {
+ "Type": "AWS::SSM::Parameter::Value<String>",
+ "Default": "/cdk-bootstrap/hnb659fds/version",
+ "Description": "Version of the CDK Bootstrap resources in this environment, automatically retrieved from SSM Parameter Store. [cdk:skip]"
+ }
+ },
+ "Rules": {
+ "CheckBootstrapVersion": {
+ "Assertions": [
+ {
+ "Assert": {
+ "Fn::Not": [
+ {
+ "Fn::Contains": [
+ [
+ "1",
+ "2",
+ "3",
+ "4",
+ "5"
+ ],
+ {
+ "Ref": "BootstrapVersion"
+ }
+ ]
+ }
+ ]
+ },
+ "AssertDescription": "CDK bootstrap stack version 6 required. Please run 'cdk bootstrap' with a recent version of the CDK CLI."
+ }
+ ]
+ }
+ }
+}
diff --git a/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/cdk.out b/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/cdk.out
@@ -1 +1 @@
-{"version":"20.0.0"}
+{"version":"30.1.0"}
diff --git a/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/integ.json b/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/integ.json
@@ -1,14 +1,12 @@
 {
- "version": "20.0.0",
+ "version": "30.1.0",
  "testCases": {
- "integ.table": {
+ "aws-cdk-glue-table-integ/DefaultTest": {
  "stacks": [
  "aws-cdk-glue"
  ],
- "diffAssets": false,
- "stackUpdateWorkflow": true
+ "assertionStack": "aws-cdk-glue-table-integ/DefaultTest/DeployAssert",
+ "assertionStackName": "awscdkgluetableintegDefaultTestDeployAssert8BFB5B70"
  }
- },
- "synthContext": {},
- "enableLookups": false
+ }
 }
diff --git a/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/manifest.json b/packages/@aws-cdk/aws-glue/test/integ.table.js.snapshot/manifest.json
@@ -1,12 +1,6 @@
 {
- "version": "20.0.0",
+ "version": "30.1.0",
  "artifacts": {
- "Tree": {
- "type": "cdk:tree",
- "properties": {
- "file": "tree.json"
- }
- },
  "aws-cdk-glue.assets": {
  "type": "cdk:asset-manifest",
  "properties": {
@@ -23,7 +17,7 @@
  "validateOnSynth": false,
  "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}",
  "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}",
- "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/eef5abdc0f1ee16e5be447f60688757df6726f3c2d1d06c136e9bbdb99d96e1f.json",
+ "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/553f4bc301289a3e09b1cd03c07892e4c62da6b35d057a57fe1613c230c27ef6.json",
  "requiresBootstrapStackVersion": 6,
  "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version",
  "additionalDependencies": [
@@ -93,12 +87,6 @@
  "data": "MyEncryptedTable981A88C6"
  }
  ],
- "/aws-cdk-glue/MyPartitionFilteredTable/Bucket/Resource": [
- {
- "type": "aws:cdk:logicalId",
- "data": "MyPartitionFilteredTableBucket6ACAA137"
- }
- ],
  "/aws-cdk-glue/MyPartitionFilteredTable/Table": [
  {
  "type": "aws:cdk:logicalId",
@@ -143,6 +131,59 @@
  ]
  },
  "displayName": "aws-cdk-glue"
+ },
+ "awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.assets": {
+ "type": "cdk:asset-manifest",
+ "properties": {
+ "file": "awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.assets.json",
+ "requiresBootstrapStackVersion": 6,
+ "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version"
+ }
+ },
+ "awscdkgluetableintegDefaultTestDeployAssert8BFB5B70": {
+ "type": "aws:cloudformation:stack",
+ "environment": "aws://unknown-account/unknown-region",
+ "properties": {
+ "templateFile": "awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.template.json",
+ "validateOnSynth": false,
+ "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}",
+ "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}",
+ "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/21fbb51d7b23f6a6c262b46a9caee79d744a3ac019fd45422d988b96d44b2a22.json",
+ "requiresBootstrapStackVersion": 6,
+ "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version",
+ "additionalDependencies": [
+ "awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.assets"
+ ],
+ "lookupRole": {
+ "arn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-lookup-role-${AWS::AccountId}-${AWS::Region}",
+ "requiresBootstrapStackVersion": 8,
+ "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version"
+ }
+ },
+ "dependencies": [
+ "awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.assets"
+ ],
+ "metadata": {
+ "/aws-cdk-glue-table-integ/DefaultTest/DeployAssert/BootstrapVersion": [
+ {
+ "type": "aws:cdk:logicalId",
+ "data": "BootstrapVersion"
+ }
+ ],
+ "/aws-cdk-glue-table-integ/DefaultTest/DeployAssert/CheckBootstrapVersion": [
+ {
+ "type": "aws:cdk:logicalId",
+ "data": "CheckBootstrapVersion"
+ }
+ ]
+ },
+ "displayName": "aws-cdk-glue-table-integ/DefaultTest/DeployAssert"
+ },
+ "Tree": {
+ "type": "cdk:tree",
+ "properties": {
+ "file": "tree.json"
+ }
  }
  }
 }