feat(glue): add ExternalTable for use with connections

packages/@aws-cdk/aws-glue-alpha/README.md

@@ -433,6 +433,26 @@ new glue.Table(this, 'MyTable', {
 
 *Note: you cannot provide a `Bucket` when creating the `Table` if you wish to use server-side encryption (`KMS`, `KMS_MANAGED` or `S3_MANAGED`)*.
 
+### Glue Connections
+
+Glue connections allow external data connections to third party databases and data warehouses. However, these connections can also be assigned to Glue Tables, allowing you to query external data sources using the Glue Data Catalog. The `connection` property of the `Table` class allows you to specify a Glue Connection to be associated with the table. Use this in combination with the `externalDataLocation` property to specify an internal destination that the connection will point to:
+
+```ts
+declare const myConnection: glue.Connection;
+declare const myDatabase: glue.Database;
+new glue.Table(this, 'MyTable', {
+ connection: myConnection,
+ externalDataLocation: 'default_db_public_example', // A table in Redshift
+ // ...
+ database: myDatabase,
+ columns: [{
+ name: 'col1',
+ type: glue.Schema.STRING,
+ }],
+ dataFormat: glue.DataFormat.JSON,
+});
+```
+
 ## Types
 
 A table's schema is a collection of columns, each of which have a `name` and a `type`. Types are recursive structures, consisting of primitive and complex types:

packages/@aws-cdk/aws-glue-alpha/lib/table.ts

@@ -6,8 +6,10 @@ import { ArnFormat, Fn, IResource, Lazy, Names, Resource, Stack } from 'aws-cdk-
 import * as cr from 'aws-cdk-lib/custom-resources';
 import { AwsCustomResource } from 'aws-cdk-lib/custom-resources';
 import { Construct } from 'constructs';
+import { IConnection } from './connection';
 import { DataFormat } from './data-format';
 import { IDatabase } from './database';
+import { S3Table } from './s3-table';
 import { Column } from './schema';
 import { StorageParameter } from './storage-parameter';
 
@@ -183,6 +185,13 @@ export interface TableProps {
  */
  readonly enablePartitionFiltering?: boolean;
 
+ /**
+ * The connection the table will use when performing reads and writes.
+ *
+ * @default - No connection
+ */
+ readonly connection?: IConnection;
+
  /**
  * The user-supplied properties for the description of the physical storage of this table. These properties help describe the format of the data that is stored within the crawled data sources.
  *
@@ -221,6 +230,8 @@ export interface TableProps {
 
 /**
  * A Glue table.
+ *
+ * @deprecate Use {@link S3Table} instead.
  */
 export class Table extends Resource implements ITable {
 
@@ -262,7 +273,7 @@ export class Table extends Resource implements ITable {
  /**
  * The type of encryption enabled for the table.
  */
- public readonly encryption: TableEncryption;
+ public readonly encryption?: TableEncryption;
 
  /**
  * The KMS key used to secure the data if `encryption` is set to `CSE-KMS` or `SSE-KMS`. Otherwise, `undefined`.
@@ -272,12 +283,12 @@ export class Table extends Resource implements ITable {
  /**
  * S3 bucket in which the table's data resides.
  */
- public readonly bucket: s3.IBucket;
+ public readonly bucket?: s3.IBucket;
 
  /**
  * S3 Key Prefix under which this table's files are stored in S3.
  */
- public readonly s3Prefix: string;
+ public readonly s3Prefix?: string;
 
  /**
  * Name of this table.
@@ -309,6 +320,11 @@ export class Table extends Resource implements ITable {
  */
  public readonly partitionIndexes?: PartitionIndex[];
 
+ /**
+ * The location of the tables' data.
+ */
+ readonly location?: string;
+
  /**
  * The tables' storage descriptor properties.
  */
@@ -331,18 +347,20 @@ export class Table extends Resource implements ITable {
 
  this.database = props.database;
  this.dataFormat = props.dataFormat;
- this.s3Prefix = props.s3Prefix ?? '';
 
  validateSchema(props.columns, props.partitionKeys);
  this.columns = props.columns;
  this.partitionKeys = props.partitionKeys;
  this.storageParameters = props.storageParameters;
 
  this.compressed = props.compressed ?? false;
+
+ this.s3Prefix = props.s3Prefix ?? '';
  const { bucket, encryption, encryptionKey } = createBucket(this, props);
  this.bucket = bucket;
  this.encryption = encryption;
  this.encryptionKey = encryptionKey;
+ this.location = `s3://${bucket.bucketName}/${this.s3Prefix}`;
 
  const tableResource = new CfnTable(this, 'Table', {
  catalogId: props.database.catalogId,
@@ -359,9 +377,10 @@ export class Table extends Resource implements ITable {
  'classification': props.dataFormat.classificationString?.value,
  'has_encrypted_data': true,
  'partition_filtering.enabled': props.enablePartitionFiltering,
+ 'connectionName': props.connection?.connectionName,
  },
  storageDescriptor: {
- location: `s3://${this.bucket.bucketName}/${this.s3Prefix}`,
+ location: this.location,
  compressed: this.compressed,
  storedAsSubDirectories: props.storedAsSubDirectories ?? false,
  columns: renderColumns(props.columns),
@@ -473,8 +492,10 @@ export class Table extends Resource implements ITable {
  */
  public grantRead(grantee: iam.IGrantable): iam.Grant {
  const ret = this.grant(grantee, readPermissions);
- if (this.encryptionKey && this.encryption === TableEncryption.CLIENT_SIDE_KMS) { this.encryptionKey.grantDecrypt(grantee); }
- this.bucket.grantRead(grantee, this.getS3PrefixForGrant());
+ if (this.bucket) {
+ if (this.encryptionKey && this.encryption === TableEncryption.CLIENT_SIDE_KMS) { this.encryptionKey.grantDecrypt(grantee); }
+ this.bucket.grantRead(grantee, this.getS3PrefixForGrant());
+ }
  return ret;
  }
 
@@ -485,8 +506,10 @@ export class Table extends Resource implements ITable {
  */
  public grantWrite(grantee: iam.IGrantable): iam.Grant {
  const ret = this.grant(grantee, writePermissions);
- if (this.encryptionKey && this.encryption === TableEncryption.CLIENT_SIDE_KMS) { this.encryptionKey.grantEncrypt(grantee); }
- this.bucket.grantWrite(grantee, this.getS3PrefixForGrant());
+ if (this.bucket) {
+ if (this.encryptionKey && this.encryption === TableEncryption.CLIENT_SIDE_KMS) { this.encryptionKey.grantEncrypt(grantee); }
+ this.bucket.grantWrite(grantee, this.getS3PrefixForGrant());
+ }
  return ret;
  }
 
@@ -497,8 +520,10 @@ export class Table extends Resource implements ITable {
  */
  public grantReadWrite(grantee: iam.IGrantable): iam.Grant {
  const ret = this.grant(grantee, [...readPermissions, ...writePermissions]);
- if (this.encryptionKey && this.encryption === TableEncryption.CLIENT_SIDE_KMS) { this.encryptionKey.grantEncryptDecrypt(grantee); }
- this.bucket.grantReadWrite(grantee, this.getS3PrefixForGrant());
+ if (this.bucket) {
+ if (this.encryptionKey && this.encryption === TableEncryption.CLIENT_SIDE_KMS) { this.encryptionKey.grantEncryptDecrypt(grantee); }
+ this.bucket.grantReadWrite(grantee, this.getS3PrefixForGrant());
+ }
  return ret;
  }
 
@@ -529,7 +554,7 @@ export class Table extends Resource implements ITable {
  });
  }
 
- private getS3PrefixForGrant() {
+ protected getS3PrefixForGrant() {
  return this.s3Prefix + '*';
  }
 }

packages/@aws-cdk/aws-glue-alpha/test/integ.table.ts

@@ -14,6 +14,16 @@ const bucket = new s3.Bucket(stack, 'DataBucket', {
  removalPolicy: cdk.RemovalPolicy.DESTROY,
 });
 
+const connection = new glue.Connection(stack, 'MyConnection', {
+ connectionName: 'my_connection',
+ type: glue.ConnectionType.JDBC,
+ properties: {
+ JDBC_CONNECTION_URL: 'jdbc:mysql://mysql.example.com:3306',
+ USERNAME: 'username',
+ PASSWORD: 'password',
+ },
+});
+
 const database = new glue.Database(stack, 'MyDatabase', {
  databaseName: 'my_database',
 });
@@ -101,6 +111,24 @@ new glue.Table(stack, 'MyPartitionFilteredTable', {
  enablePartitionFiltering: true,
 });
 
+new glue.Table(stack, 'MyTableWithConnection', {
+ database,
+ bucket,
+ tableName: 'connection_table',
+ columns,
+ dataFormat: glue.DataFormat.JSON,
+ connection,
+});
+
+new glue.Table(stack, 'MyTableWithCustomLocation', {
+ database,
+ bucket,
+ tableName: 'custom_location_table',
+ columns,
+ dataFormat: glue.DataFormat.JSON,
+ externalDataLocation: 'default_db.public.test',
+});
+
 new glue.Table(stack, 'MyTableWithStorageDescriptorParameters', {
  database,
  bucket,

packages/@aws-cdk/aws-glue-alpha/test/table.test.ts

@@ -95,7 +95,8 @@ test('partitioned JSON table', () => {
  });
  expect(table.encryption).toEqual(glue.TableEncryption.S3_MANAGED);
  expect(table.encryptionKey).toEqual(undefined);
- expect(table.bucket.encryptionKey).toEqual(undefined);
+ expect(table.bucket).not.toEqual(undefined);
+ expect(table.bucket?.encryptionKey).toEqual(undefined);
 
  Template.fromStack(tableStack).hasResourceProperties('AWS::Glue::Table', {
  CatalogId: {
@@ -163,7 +164,7 @@ test('compressed table', () => {
  dataFormat: glue.DataFormat.JSON,
  });
  expect(table.encryptionKey).toEqual(undefined);
- expect(table.bucket.encryptionKey).toEqual(undefined);
+ expect(table.bucket?.encryptionKey).toEqual(undefined);
 
  Template.fromStack(stack).hasResourceProperties('AWS::Glue::Table', {
  CatalogId: {
@@ -246,7 +247,7 @@ test('encrypted table: SSE-S3', () => {
  });
  expect(table.encryption).toEqual(glue.TableEncryption.S3_MANAGED);
  expect(table.encryptionKey).toEqual(undefined);
- expect(table.bucket.encryptionKey).toEqual(undefined);
+ expect(table.bucket?.encryptionKey).toEqual(undefined);
 
  Template.fromStack(stack).hasResourceProperties('AWS::Glue::Table', {
  CatalogId: {
@@ -320,7 +321,7 @@ test('encrypted table: SSE-KMS (implicitly created key)', () => {
  dataFormat: glue.DataFormat.JSON,
  });
  expect(table.encryption).toEqual(glue.TableEncryption.KMS);
- expect(table.encryptionKey).toEqual(table.bucket.encryptionKey);
+ expect(table.encryptionKey).toEqual(table.bucket?.encryptionKey);
 
  Template.fromStack(stack).hasResourceProperties('AWS::KMS::Key', {
  Description: 'Created by Default/Table/Bucket',
@@ -408,7 +409,7 @@ test('encrypted table: SSE-KMS (explicitly created key)', () => {
  dataFormat: glue.DataFormat.JSON,
  });
  expect(table.encryption).toEqual(glue.TableEncryption.KMS);
- expect(table.encryptionKey).toEqual(table.bucket.encryptionKey);
+ expect(table.encryptionKey).toEqual(table.bucket?.encryptionKey);
  expect(table.encryptionKey).not.toEqual(undefined);
 
  Template.fromStack(stack).hasResourceProperties('AWS::KMS::Key', {
@@ -494,7 +495,7 @@ test('encrypted table: SSE-KMS_MANAGED', () => {
  });
  expect(table.encryption).toEqual(glue.TableEncryption.KMS_MANAGED);
  expect(table.encryptionKey).toEqual(undefined);
- expect(table.bucket.encryptionKey).toEqual(undefined);
+ expect(table.bucket?.encryptionKey).toEqual(undefined);
 
  Template.fromStack(stack).hasResourceProperties('AWS::S3::Bucket', {
  BucketEncryption: {
@@ -569,7 +570,7 @@ test('encrypted table: CSE-KMS (implicitly created key)', () => {
  });
  expect(table.encryption).toEqual(glue.TableEncryption.CLIENT_SIDE_KMS);
  expect(table.encryptionKey).not.toEqual(undefined);
- expect(table.bucket.encryptionKey).toEqual(undefined);
+ expect(table.bucket?.encryptionKey).toEqual(undefined);
 
  Template.fromStack(stack).resourceCountIs('AWS::KMS::Key', 1);
 
@@ -638,7 +639,7 @@ test('encrypted table: CSE-KMS (explicitly created key)', () => {
  });
  expect(table.encryption).toEqual(glue.TableEncryption.CLIENT_SIDE_KMS);
  expect(table.encryptionKey).not.toEqual(undefined);
- expect(table.bucket.encryptionKey).toEqual(undefined);
+ expect(table.bucket?.encryptionKey).toEqual(undefined);
 
  Template.fromStack(stack).hasResourceProperties('AWS::KMS::Key', {
  Description: 'MyKey',
@@ -711,7 +712,7 @@ test('encrypted table: CSE-KMS (explicitly passed bucket and key)', () => {
  });
  expect(table.encryption).toEqual(glue.TableEncryption.CLIENT_SIDE_KMS);
  expect(table.encryptionKey).not.toEqual(undefined);
- expect(table.bucket.encryptionKey).toEqual(undefined);
+ expect(table.bucket?.encryptionKey).toEqual(undefined);
 
  Template.fromStack(stack).hasResourceProperties('AWS::KMS::Key', {
  Description: 'MyKey',
@@ -1654,6 +1655,63 @@ test('storage descriptor parameters', () => {
  });
 });
 
+test('can associate a connection with the glue table', () => {
+ const app = new cdk.App();
+ const stack = new cdk.Stack(app, 'Stack');
+ const database = new glue.Database(stack, 'Database');
+ const connection = new glue.Connection(stack, 'Connection', {
+ connectionName: 'my_connection',
+ type: glue.ConnectionType.JDBC,
+ properties: {
+ JDBC_CONNECTION_URL: 'jdbc:server://server:443/connection',
+ USERNAME: 'username',
+ PASSWORD: 'password',
+ },
+ });
+ new glue.Table(stack, 'Table', {
+ database,
+ tableName: 'my_table',
+ columns: [{
+ name: 'col',
+ type: glue.Schema.STRING,
+ }],
+ dataFormat: glue.DataFormat.JSON,
+ connection,
+ });
+
+ Template.fromStack(stack).hasResourceProperties('AWS::Glue::Table', {
+ TableInput: {
+ Parameters: {
+ connectionName: { Ref: 'Connection89AD5CF5' },
+ },
+ },
+ });
+});
+
+test('can associate an external location with the glue table', () => {
+ const app = new cdk.App();
+ const stack = new cdk.Stack(app, 'Stack');
+ const database = new glue.Database(stack, 'Database');
+ new glue.Table(stack, 'Table', {
+ database,
+ tableName: 'my_table',
+ columns: [{
+ name: 'col',
+ type: glue.Schema.STRING,
+ }],
+ dataFormat: glue.DataFormat.JSON,
+ externalDataLocation: 'default_db.public.test',
+ });
+
+ Template.fromStack(stack).hasResourceProperties('AWS::Glue::Table', {
+ TableInput: {
+ StorageDescriptor: {
+ Location: 'default_db.public.test',
+ },
+ },
+ });
+});
+
 function createTable(props: Pick<glue.TableProps, Exclude<keyof glue.TableProps, 'database' | 'dataFormat'>>): void {
  const stack = new cdk.Stack();
  new glue.Table(stack, 'table', {