feat(pipes-sources): add Kinesis and DynamoDB

packages/@aws-cdk/aws-pipes-alpha/lib/pipe.ts

@@ -5,7 +5,7 @@ import { Construct } from 'constructs';
 import { IEnrichment } from './enrichment';
 import { IFilter } from './filter';
 import { ILogDestination, IncludeExecutionData, LogLevel } from './logs';
-import { ISource } from './source';
+import { ISource, SourceWithDeadLetterTarget } from './source';
 import { ITarget } from './target';
 
 /**
@@ -233,6 +233,16 @@ export class Pipe extends PipeBase {
  */
  const source = props.source.bind(this);
  props.source.grantRead(this.pipeRole);
+
+ /**
+ * An optional dead-letter queue stores any events that are not successfully delivered to
+ * a target after all retry attempts are exhausted. The IAM role needs permission to write
+ * events to the dead-letter queue, either an SQS queue or SNS topic.
+ */
+ if (SourceWithDeadLetterTarget.isSourceWithDeadLetterTarget(props.source)) {
+ props.source.grantPush(this.pipeRole, props.source.deadLetterTarget);
+ }
+
  // Add the filter criteria to the source parameters
  const sourceParameters : CfnPipe.PipeSourceParametersProperty= {
  ...source.sourceParameters,

packages/@aws-cdk/aws-pipes-alpha/lib/source.ts

@@ -1,6 +1,9 @@
 import { IRole } from 'aws-cdk-lib/aws-iam';
 import { CfnPipe } from 'aws-cdk-lib/aws-pipes';
+import { ITopic, Topic } from 'aws-cdk-lib/aws-sns';
+import { IQueue, Queue } from 'aws-cdk-lib/aws-sqs';
 import { IPipe } from './pipe';
+
 /**
  * Source properties.
  *
@@ -100,3 +103,54 @@ export interface ISource {
  */
  grantRead(grantee: IRole): void;
 }
+
+/**
+ * Sources that support a dead-letter target.
+ */
+export abstract class SourceWithDeadLetterTarget implements ISource {
+ /**
+ * Determines if the source is an instance of SourceWithDeadLetterTarget.
+ */
+ public static isSourceWithDeadLetterTarget(source: ISource): source is SourceWithDeadLetterTarget {
+ return (source as SourceWithDeadLetterTarget).deadLetterTarget !== undefined;
+ }
+ /**
+ * The ARN of the source resource.
+ */
+ readonly sourceArn: string;
+ /**
+ * The dead-letter SQS queue or SNS topic.
+ */
+ readonly deadLetterTarget?: IQueue | ITopic;
+
+ constructor(sourceArn: string, deadLetterTarget?: IQueue | ITopic) {
+ this.sourceArn = sourceArn;
+ this.deadLetterTarget = deadLetterTarget;
+ }
+
+ abstract bind(pipe: IPipe): SourceConfig;
+ abstract grantRead(grantee: IRole): void;
+
+ /**
+ * Grants the pipe role permission to publish to the dead-letter target.
+ */
+ public grantPush(grantee: IRole, deadLetterTarget?: IQueue | ITopic) {
+ if (deadLetterTarget instanceof Queue) {
+ deadLetterTarget.grantSendMessages(grantee);
+ } else if (deadLetterTarget instanceof Topic) {
+ deadLetterTarget.grantPublish(grantee);
+ }
+ }
+
+ /**
+ * Retrieves the ARN from the dead-letter SQS queue or SNS topic.
+ */
+ protected getDeadLetterTargetArn(deadLetterTarget?: IQueue | ITopic): string | undefined {
+ if (deadLetterTarget instanceof Queue) {
+ return deadLetterTarget.queueArn;
+ } else if (deadLetterTarget instanceof Topic) {
+ return deadLetterTarget.topicArn;
+ }
+ return undefined;
+ }
+}

packages/@aws-cdk/aws-pipes-alpha/test/pipe.test.ts

@@ -1,8 +1,9 @@
 import { App, Stack } from 'aws-cdk-lib';
-
 import { Template } from 'aws-cdk-lib/assertions';
 import { Role, ServicePrincipal } from 'aws-cdk-lib/aws-iam';
-import { TestEnrichment, TestSource, TestTarget } from './test-classes';
+import { Topic } from 'aws-cdk-lib/aws-sns';
+import { Queue } from 'aws-cdk-lib/aws-sqs';
+import { TestEnrichment, TestSource, TestSourceWithDeadLetterTarget, TestTarget } from './test-classes';
 import { DesiredState, IEnrichment, ILogDestination, IPipe, ISource, ITarget, IncludeExecutionData, LogDestinationConfig, LogLevel, Pipe } from '../lib';
 
 class TestLogDestination implements ILogDestination {
@@ -186,6 +187,75 @@ describe('Pipe', () => {
  );
 
  });
+
+ test('grantPush is called for sources with an SNS topic DLQ', () => {
+ // WHEN
+ const topic = new Topic(stack, 'MyTopic');
+ const sourceWithDeadLetterTarget = new TestSourceWithDeadLetterTarget(topic);
+
+ new Pipe(stack, 'TestPipe', {
+ pipeName: 'TestPipe',
+ source: sourceWithDeadLetterTarget,
+ target,
+ });
+
+ const template = Template.fromStack(stack);
+
+ // THEN
+ template.hasResource('AWS::IAM::Policy', {
+ Properties: {
+ Roles: [{
+ Ref: 'TestPipeRole0FD00B2B',
+ }],
+ PolicyDocument: {
+ Statement: [{
+ Action: 'sns:Publish',
+ Resource: {
+ Ref: 'MyTopic86869434',
+ },
+ }],
+ },
+ },
+ });
+ });
+
+ test('grantPush is called for sources with an SQS queue DLQ', () => {
+ // WHEN
+ const queue = new Queue(stack, 'MyQueue');
+ const sourceWithDeadLetterTarget = new TestSourceWithDeadLetterTarget(queue);
+
+ new Pipe(stack, 'TestPipe', {
+ pipeName: 'TestPipe',
+ source: sourceWithDeadLetterTarget,
+ target,
+ });
+
+ const template = Template.fromStack(stack);
+
+ // THEN
+ template.hasResource('AWS::IAM::Policy', {
+ Properties: {
+ Roles: [{
+ Ref: 'TestPipeRole0FD00B2B',
+ }],
+ PolicyDocument: {
+ Statement: [{
+ Action: [
+ 'sqs:SendMessage',
+ 'sqs:GetQueueAttributes',
+ 'sqs:GetQueueUrl',
+ ],
+ Resource: {
+ 'Fn::GetAtt': [
+ 'MyQueueE6CA6235',
+ 'Arn',
+ ],
+ },
+ }],
+ },
+ },
+ });
+ });
  });
 
  describe('target', () => {

packages/@aws-cdk/aws-pipes-alpha/test/test-classes.ts

@@ -1,5 +1,19 @@
+import { Role } from 'aws-cdk-lib/aws-iam';
 import { CfnPipe } from 'aws-cdk-lib/aws-pipes';
-import { EnrichmentParametersConfig, IEnrichment, ILogDestination, IPipe, ISource, ITarget, LogDestinationConfig, SourceConfig, SourceParameters } from '../lib';
+import { ITopic, Topic } from 'aws-cdk-lib/aws-sns';
+import { IQueue, Queue } from 'aws-cdk-lib/aws-sqs';
+import {
+ EnrichmentParametersConfig,
+ IEnrichment,
+ ILogDestination,
+ IPipe,
+ ISource,
+ ITarget,
+ LogDestinationConfig,
+ SourceConfig,
+ SourceParameters,
+ SourceWithDeadLetterTarget,
+} from '../lib';
 
 export class TestSource implements ISource {
  readonly sourceArn = 'source-arn';
@@ -19,6 +33,30 @@ export class TestSource implements ISource {
  }
 }
 
+export class TestSourceWithDeadLetterTarget extends SourceWithDeadLetterTarget {
+ deadLetterTarget?: IQueue | ITopic;
+ public grantRead = jest.fn();
+
+ constructor(deadLetterTarget: IQueue | ITopic) {
+ super('source-arn', deadLetterTarget);
+ this.deadLetterTarget = deadLetterTarget;
+ }
+
+ grantPush(grantee: Role, deadLetterTarget?: IQueue | ITopic) {
+ if (deadLetterTarget instanceof Queue) {
+ deadLetterTarget.grantSendMessages(grantee);
+ } else if (deadLetterTarget instanceof Topic) {
+ deadLetterTarget.grantPublish(grantee);
+ }
+ }
+
+ bind(_pipe: IPipe): SourceConfig {
+ return {
+ sourceParameters: {},
+ };
+ }
+}
+
 export class TestTarget implements ITarget {
  readonly targetArn: string = 'target-arn';
  private targetParameters: CfnPipe.PipeTargetParametersProperty = {};

packages/@aws-cdk/aws-pipes-sources-alpha/README.md

@@ -61,3 +61,44 @@ const pipe = new pipes.Pipe(this, 'Pipe', {
 });
 ```
 
+### Amazon Kinesis
+
+A Kinesis stream can be used as a source for a pipe. The stream will be polled for new messages and the messages will be sent to the pipe.
+
+```ts
+declare const sourceStream: kinesis.Stream;
+declare const targetQueue: sqs.Queue;
+
+const pipeSource = new sources.KinesisSource(sourceStream, {
+ startingPosition: sources.KinesisStartingPosition.LATEST,
+});
+
+const pipe = new pipes.Pipe(this, 'Pipe', {
+ source: pipeSource,
+ target: new SomeTarget(targetQueue)
+});
+```
+
+### Amazon DynamoDB
+
+A DynamoDB stream can be used as a source for a pipe. The stream will be polled for new messages and the messages will be sent to the pipe.
+
+```ts
+const table = new ddb.TableV2(this, 'MyTable', {
+ partitionKey: {
+ name: 'id',
+ type: ddb.AttributeType.STRING,
+ },
+ dynamoStream: ddb.StreamViewType.NEW_IMAGE,
+});
+declare const targetQueue: sqs.Queue;
+
+const pipeSource = new sources.DynamoDBSource(table, {
+ startingPosition: sources.DynamoDBStartingPosition.LATEST,
+});
+
+const pipe = new pipes.Pipe(this, 'Pipe', {
+ source: pipeSource,
+ target: new SomeTarget(targetQueue)
+});
+```

packages/@aws-cdk/aws-pipes-sources-alpha/lib/dynamodb.ts

@@ -0,0 +1,58 @@
+import { IPipe, SourceConfig } from '@aws-cdk/aws-pipes-alpha';
+import { ITableV2 } from 'aws-cdk-lib/aws-dynamodb';
+import { IRole } from 'aws-cdk-lib/aws-iam';
+import { DynamoDBStartingPosition } from './enums';
+import { StreamSource, StreamSourceParameters } from './streamSource';
+
+/**
+ * Parameters for the DynamoDB source.
+ */
+export interface DynamoDBSourceParameters extends StreamSourceParameters {
+ /**
+ * The position in a stream from which to start reading.
+ *
+ * @see https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-pipes-pipe-pipesourcedynamodbstreamparameters.html#cfn-pipes-pipe-pipesourcedynamodbstreamparameters-startingposition
+ */
+ readonly startingPosition: DynamoDBStartingPosition;
+}
+
+/**
+ * A source that reads from an DynamoDB stream.
+ */
+export class DynamoDBSource extends StreamSource {
+ private readonly table: ITableV2;
+ private readonly startingPosition: DynamoDBStartingPosition;
+ private readonly deadLetterTargetArn?: string;
+
+ constructor(table: ITableV2, parameters: DynamoDBSourceParameters) {
+ if (table.tableStreamArn === undefined) {
+ throw new Error('Table does not have a stream defined, cannot create pipes source');
+ }
+
+ super(table.tableStreamArn, parameters);
+ this.table = table;
+ this.startingPosition = parameters.startingPosition;
+ this.deadLetterTargetArn = this.getDeadLetterTargetArn(this.deadLetterTarget);
+ }
+
+ bind(_pipe: IPipe): SourceConfig {
+ return {
+ sourceParameters: {
+ dynamoDbStreamParameters: {
+ batchSize: this.sourceParameters.batchSize,
+ deadLetterConfig: this.deadLetterTargetArn ? { arn: this.deadLetterTargetArn } : undefined,
+ maximumBatchingWindowInSeconds: this.sourceParameters.maximumBatchingWindow?.toSeconds(),
+ maximumRecordAgeInSeconds: this.sourceParameters.maximumRecordAge?.toSeconds(),
+ maximumRetryAttempts: this.sourceParameters.maximumRetryAttempts,
+ onPartialBatchItemFailure: this.sourceParameters.onPartialBatchItemFailure,
+ parallelizationFactor: this.sourceParameters.parallelizationFactor,
+ startingPosition: this.startingPosition,
+ },
+ },
+ };
+ }
+
+ grantRead(grantee: IRole): void {
+ this.table.grantStreamRead(grantee);
+ }
+}

packages/@aws-cdk/aws-pipes-sources-alpha/lib/enums.ts

@@ -0,0 +1,47 @@
+/**
+ * Define how to handle item process failures.
+ */
+export enum OnPartialBatchItemFailure {
+ /**
+ * EventBridge halves each batch and will retry each half until all the
+ * records are processed or there is one failed message left in the batch.
+ */
+ AUTOMATIC_BISECT = 'AUTOMATIC_BISECT',
+}
+
+/**
+ * The position in a Kinesis stream from which to start reading.
+ */
+export enum KinesisStartingPosition {
+ /**
+ * Start streaming at the last untrimmed record in the shard,
+ * which is the oldest data record in the shard.
+ */
+ TRIM_HORIZON = 'TRIM_HORIZON',
+ /**
+ * Start streaming just after the most recent record in the shard,
+ * so that you always read the most recent data in the shard.
+ */
+ LATEST = 'LATEST',
+ /**
+ * Start streaming from the position denoted by the time stamp
+ * specified in the `startingPositionTimestamp` field.
+ */
+ AT_TIMESTAMP = 'AT_TIMESTAMP',
+}
+
+/**
+ * The position in a DynamoDB stream from which to start reading.
+ */
+export enum DynamoDBStartingPosition {
+ /**
+ * Start reading at the last (untrimmed) stream record,
+ * which is the oldest record in the shard.
+ */
+ TRIM_HORIZON = 'TRIM_HORIZON',
+ /**
+ * Start reading just after the most recent stream record in the shard,
+ * so that you always read the most recent data in the shard.
+ */
+ LATEST = 'LATEST',
+}

packages/@aws-cdk/aws-pipes-sources-alpha/lib/index.ts

@@ -1 +1,5 @@
+export * from './dynamodb';
+export * from './enums';
+export * from './kinesis';
 export * from './sqs';
+export * from './streamSource';