diff --git a/packages/@aws-cdk/aws-stepfunctions-tasks/README.md b/packages/@aws-cdk/aws-stepfunctions-tasks/README.md index 2f5d463067a35..59adc3517e704 100644 --- a/packages/@aws-cdk/aws-stepfunctions-tasks/README.md +++ b/packages/@aws-cdk/aws-stepfunctions-tasks/README.md @@ -60,6 +60,10 @@ This module is part of the [AWS Cloud Development Kit](https://github.com/aws/aw - [Cancel Step](#cancel-step) - [Modify Instance Fleet](#modify-instance-fleet) - [Modify Instance Group](#modify-instance-group) + - [EMR on EKS](#emr-on-eks) + - [Create Virtual Cluster](#create-virtual-cluster) + - [Delete Virtual Cluster](#delete-virtual-cluster) + - [Start Job Run](#start-job-run) - [EKS](#eks) - [Call](#call) - [EventBridge](#eventbridge) @@ -783,6 +787,169 @@ new tasks.EmrModifyInstanceGroupByName(this, 'Task', { }); ``` +## EMR on EKS + +Step Functions supports Amazon EMR on EKS through the service integration pattern. +The service integration APIs correspond to Amazon EMR on EKS APIs, but differ in the parameters that are used. + +[Read more](https://docs.aws.amazon.com/step-functions/latest/dg/connect-emr-eks.html) about the differences when using these service integrations. + +[Setting up](https://docs.aws.amazon.com/emr/latest/EMR-on-EKS-DevelopmentGuide/setting-up.html) the EKS cluster is required. + +### Create Virtual Cluster + +The [CreateVirtualCluster](https://docs.aws.amazon.com/emr-on-eks/latest/APIReference/API_CreateVirtualCluster.html) API creates a single virtual cluster that's mapped to a single Kubernetes namespace. + +The EKS cluster containing the Kubernetes namespace where the virtual cluster will be mapped can be passed in from the task input. + +```ts +new tasks.EmrContainersCreateVirtualCluster(this, 'Create a Virtual Cluster', { + eksCluster: tasks.EksClusterInput.fromTaskInput(sfn.TaskInput.fromText('clusterId')), +}); +``` + +The EKS cluster can also be passed in directly. + +```ts +import * as eks from '@aws-cdk/aws-eks'; + +declare const eksCluster: eks.Cluster; + +new tasks.EmrContainersCreateVirtualCluster(this, 'Create a Virtual Cluster', { + eksCluster: tasks.EksClusterInput.fromCluster(eksCluster), +}); +``` + +By default, the Kubernetes namespace that a virtual cluster maps to is "default", but a specific namespace within an EKS cluster can be selected. + +```ts +new tasks.EmrContainersCreateVirtualCluster(this, 'Create a Virtual Cluster', { + eksCluster: tasks.EksClusterInput.fromTaskInput(sfn.TaskInput.fromText('clusterId')), + eksNamespace: 'specified-namespace', +}); +``` + +### Delete Virtual Cluster + +The [DeleteVirtualCluster](https://docs.aws.amazon.com/emr-on-eks/latest/APIReference/API_DeleteVirtualCluster.html) API deletes a virtual cluster. + +```ts +new tasks.EmrContainersDeleteVirtualCluster(this, 'Delete a Virtual Cluster', { + virtualClusterId: sfn.TaskInput.fromJsonPathAt('$.virtualCluster'), +}); +``` + +### Start Job Run + +The [StartJobRun](https://docs.aws.amazon.com/emr-on-eks/latest/APIReference/API_StartJobRun.html) API starts a job run. A job is a unit of work that you submit to Amazon EMR on EKS for execution. The work performed by the job can be defined by a Spark jar, PySpark script, or SparkSQL query. A job run is an execution of the job on the virtual cluster. + +Required setup: + + - If not done already, follow the [steps](https://docs.aws.amazon.com/emr/latest/EMR-on-EKS-DevelopmentGuide/setting-up.html) to setup EMR on EKS and [create an EKS Cluster](https://docs.aws.amazon.com/cdk/api/latest/docs/aws-eks-readme.html#quick-start). + - Enable [Cluster access](https://docs.aws.amazon.com/emr/latest/EMR-on-EKS-DevelopmentGuide/setting-up-cluster-access.html) + - Enable [IAM Role access](https://docs.aws.amazon.com/emr/latest/EMR-on-EKS-DevelopmentGuide/setting-up-enable-IAM.html) + +The following actions must be performed if the virtual cluster ID is supplied from the task input. Otherwise, if it is supplied statically in the state machine definition, these actions will be done automatically. + + - Create an [IAM role](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-iam.Role.html) + - Update the [Role Trust Policy](https://docs.aws.amazon.com/emr/latest/EMR-on-EKS-DevelopmentGuide/setting-up-trust-policy.html) of the Job Execution Role. + +The job can be configured with spark submit parameters: + +```ts +new tasks.EmrContainersStartJobRun(this, 'EMR Containers Start Job Run', { + virtualCluster: tasks.VirtualClusterInput.fromVirtualClusterId('de92jdei2910fwedz'), + releaseLabel: tasks.ReleaseLabel.EMR_6_2_0, + jobDriver: { + sparkSubmitJobDriver: { + entryPoint: sfn.TaskInput.fromText('local:///usr/lib/spark/examples/src/main/python/pi.py'), + sparkSubmitParameters: '--conf spark.executor.instances=2 --conf spark.executor.memory=2G --conf spark.executor.cores=2 --conf spark.driver.cores=1', + }, + }, +}); +``` + +Configuring the job can also be done via application configuration: + +```ts +new tasks.EmrContainersStartJobRun(this, 'EMR Containers Start Job Run', { + virtualCluster: tasks.VirtualClusterInput.fromVirtualClusterId('de92jdei2910fwedz'), + releaseLabel: tasks.ReleaseLabel.EMR_6_2_0, + jobName: 'EMR-Containers-Job', + jobDriver: { + sparkSubmitJobDriver: { + entryPoint: sfn.TaskInput.fromText('local:///usr/lib/spark/examples/src/main/python/pi.py'), + }, + }, + applicationConfig: [{ + classification: tasks.Classification.SPARK_DEFAULTS, + properties: { + 'spark.executor.instances': '1', + 'spark.executor.memory': '512M', + }, + }], +}); +``` + +Job monitoring can be enabled if `monitoring.logging` is set true. This automatically generates an S3 bucket and CloudWatch logs. + +```ts +new tasks.EmrContainersStartJobRun(this, 'EMR Containers Start Job Run', { + virtualCluster: tasks.VirtualClusterInput.fromVirtualClusterId('de92jdei2910fwedz'), + releaseLabel: tasks.ReleaseLabel.EMR_6_2_0, + jobDriver: { + sparkSubmitJobDriver: { + entryPoint: sfn.TaskInput.fromText('local:///usr/lib/spark/examples/src/main/python/pi.py'), + sparkSubmitParameters: '--conf spark.executor.instances=2 --conf spark.executor.memory=2G --conf spark.executor.cores=2 --conf spark.driver.cores=1', + }, + }, + monitoring: { + logging: true, + }, +}); +``` + +Otherwise, providing monitoring for jobs with existing log groups and log buckets is also available. + +```ts +import * as logs from '@aws-cdk/aws-logs'; + +const logGroup = new logs.LogGroup(this, 'Log Group'); +const logBucket = new s3.Bucket(this, 'S3 Bucket') + +new tasks.EmrContainersStartJobRun(this, 'EMR Containers Start Job Run', { + virtualCluster: tasks.VirtualClusterInput.fromVirtualClusterId('de92jdei2910fwedz'), + releaseLabel: tasks.ReleaseLabel.EMR_6_2_0, + jobDriver: { + sparkSubmitJobDriver: { + entryPoint: sfn.TaskInput.fromText('local:///usr/lib/spark/examples/src/main/python/pi.py'), + sparkSubmitParameters: '--conf spark.executor.instances=2 --conf spark.executor.memory=2G --conf spark.executor.cores=2 --conf spark.driver.cores=1', + }, + }, + monitoring: { + logGroup: logGroup, + logBucket: logBucket, + }, +}); +``` + +Users can provide their own existing Job Execution Role. + +```ts +new tasks.EmrContainersStartJobRun(this, 'EMR Containers Start Job Run', { + virtualCluster:tasks.VirtualClusterInput.fromTaskInput(sfn.TaskInput.fromJsonPathAt('$.VirtualClusterId')), + releaseLabel: tasks.ReleaseLabel.EMR_6_2_0, + jobName: 'EMR-Containers-Job', + executionRole: iam.Role.fromRoleArn(this, 'Job-Execution-Role', 'arn:aws:iam::xxxxxxxxxxxx:role/JobExecutionRole'), + jobDriver: { + sparkSubmitJobDriver: { + entryPoint: sfn.TaskInput.fromText('local:///usr/lib/spark/examples/src/main/python/pi.py'), + sparkSubmitParameters: '--conf spark.executor.instances=2 --conf spark.executor.memory=2G --conf spark.executor.cores=2 --conf spark.driver.cores=1', + }, + }, +}); +``` + ## EKS Step Functions supports Amazon EKS through the service integration pattern. diff --git a/packages/@aws-cdk/aws-stepfunctions-tasks/lib/emrcontainers/create-virtual-cluster.ts b/packages/@aws-cdk/aws-stepfunctions-tasks/lib/emrcontainers/create-virtual-cluster.ts new file mode 100644 index 0000000000000..cb9458ef01e44 --- /dev/null +++ b/packages/@aws-cdk/aws-stepfunctions-tasks/lib/emrcontainers/create-virtual-cluster.ts @@ -0,0 +1,146 @@ +import * as eks from '@aws-cdk/aws-eks'; +import * as iam from '@aws-cdk/aws-iam'; +import * as sfn from '@aws-cdk/aws-stepfunctions'; +import { Stack } from '@aws-cdk/core'; +import { Construct } from 'constructs'; +import { integrationResourceArn, validatePatternSupported } from '../private/task-utils'; + +/** + * Class for supported types of EMR Containers' Container Providers + */ +enum ContainerProviderTypes { + + /** + * Supported container provider type for a EKS Cluster + */ + EKS = 'EKS' +} + +/** + * Class that supports methods which return the EKS cluster name depending on input type. + */ +export class EksClusterInput { + + /** + * Specify an existing EKS Cluster as the name for this Cluster + */ + static fromCluster(cluster: eks.ICluster): EksClusterInput { + return new EksClusterInput(cluster.clusterName); + } + + /** + * Specify a Task Input as the name for this Cluster + */ + static fromTaskInput(taskInput: sfn.TaskInput): EksClusterInput { + return new EksClusterInput(taskInput.value); + } + + /** + * Initializes the clusterName + * + * @param clusterName The name of the EKS Cluster + */ + private constructor(readonly clusterName: string) { } +} + +/** + * Properties to define a EMR Containers CreateVirtualCluster Task on an EKS cluster + */ +export interface EmrContainersCreateVirtualClusterProps extends sfn.TaskStateBaseProps { + + /** + * EKS Cluster or task input that contains the name of the cluster + */ + readonly eksCluster: EksClusterInput; + + /** + * The namespace of an EKS cluster + * + * @default - 'default' + */ + readonly eksNamespace?: string; + + /** + * Name of the virtual cluster that will be created. + * + * @default - the name of the state machine execution that runs this task and state name + */ + readonly virtualClusterName?: string; + + /** + * The tags assigned to the virtual cluster + * + * @default {} + */ + readonly tags?: { [key: string]: string }; +} + +/** + * Task that creates an EMR Containers virtual cluster from an EKS cluster + * + * @see https://docs.aws.amazon.com/step-functions/latest/dg/connect-emr-eks.html + */ +export class EmrContainersCreateVirtualCluster extends sfn.TaskStateBase { + + private static readonly SUPPORTED_INTEGRATION_PATTERNS: sfn.IntegrationPattern[] = [ + sfn.IntegrationPattern.REQUEST_RESPONSE, + ]; + + protected readonly taskMetrics?: sfn.TaskMetricsConfig; + protected readonly taskPolicies?: iam.PolicyStatement[]; + + private readonly integrationPattern: sfn.IntegrationPattern; + + constructor(scope: Construct, id: string, private readonly props: EmrContainersCreateVirtualClusterProps) { + super(scope, id, props); + this.integrationPattern = props.integrationPattern ?? sfn.IntegrationPattern.REQUEST_RESPONSE; + validatePatternSupported(this.integrationPattern, EmrContainersCreateVirtualCluster.SUPPORTED_INTEGRATION_PATTERNS); + + this.taskPolicies = this.createPolicyStatements(); + } + + /** + * @internal + */ + protected _renderTask(): any { + return { + Resource: integrationResourceArn('emr-containers', 'createVirtualCluster', this.integrationPattern), + Parameters: sfn.FieldUtils.renderObject({ + Name: this.props.virtualClusterName ?? sfn.JsonPath.stringAt('States.Format(\'{}/{}\', $$.Execution.Name, $$.State.Name)'), + ContainerProvider: { + Id: this.props.eksCluster.clusterName, + Info: { + EksInfo: { + Namespace: this.props.eksNamespace ?? 'default', + }, + }, + Type: ContainerProviderTypes.EKS, + }, + Tags: this.props.tags, + }), + }; + }; + + private createPolicyStatements(): iam.PolicyStatement[] { + return [ + new iam.PolicyStatement({ + resources: ['*'], // We need * permissions for creating a virtual cluster https://docs.aws.amazon.com/emr/latest/EMR-on-EKS-DevelopmentGuide/setting-up-iam.html + actions: ['emr-containers:CreateVirtualCluster'], + }), + new iam.PolicyStatement({ + resources: [ + Stack.of(this).formatArn({ + service: 'iam', + region: '', + resource: 'role/aws-service-role/emr-containers.amazonaws.com', + resourceName: 'AWSServiceRoleForAmazonEMRContainers', + }), + ], + actions: ['iam:CreateServiceLinkedRole'], + conditions: { + StringLike: { 'iam:AWSServiceName': 'emr-containers.amazonaws.com' }, + }, + }), + ]; + } +} diff --git a/packages/@aws-cdk/aws-stepfunctions-tasks/lib/emrcontainers/delete-virtual-cluster.ts b/packages/@aws-cdk/aws-stepfunctions-tasks/lib/emrcontainers/delete-virtual-cluster.ts new file mode 100644 index 0000000000000..b4e0e9c45e2ce --- /dev/null +++ b/packages/@aws-cdk/aws-stepfunctions-tasks/lib/emrcontainers/delete-virtual-cluster.ts @@ -0,0 +1,74 @@ +import * as iam from '@aws-cdk/aws-iam'; +import * as sfn from '@aws-cdk/aws-stepfunctions'; +import * as cdk from '@aws-cdk/core'; +import { Construct } from 'constructs'; +import { integrationResourceArn, validatePatternSupported } from '../private/task-utils'; + +/** + * Properties to define a EMR Containers DeleteVirtualCluster Task + */ +export interface EmrContainersDeleteVirtualClusterProps extends sfn.TaskStateBaseProps { + + /** + * The ID of the virtual cluster that will be deleted. + */ + readonly virtualClusterId: sfn.TaskInput; +} + +/** + * Deletes an EMR Containers virtual cluster as a Task. + * + * @see https://docs.amazonaws.cn/en_us/step-functions/latest/dg/connect-emr-eks.html + */ +export class EmrContainersDeleteVirtualCluster extends sfn.TaskStateBase { + + private static readonly SUPPORTED_INTEGRATION_PATTERNS: sfn.IntegrationPattern[] = [ + sfn.IntegrationPattern.REQUEST_RESPONSE, + sfn.IntegrationPattern.RUN_JOB, + ]; + + protected readonly taskMetrics?: sfn.TaskMetricsConfig; + protected readonly taskPolicies?: iam.PolicyStatement[]; + + private readonly integrationPattern: sfn.IntegrationPattern; + + constructor(scope: Construct, id: string, private readonly props: EmrContainersDeleteVirtualClusterProps) { + super(scope, id, props); + this.integrationPattern = props.integrationPattern ?? sfn.IntegrationPattern.REQUEST_RESPONSE; + + validatePatternSupported(this.integrationPattern, EmrContainersDeleteVirtualCluster.SUPPORTED_INTEGRATION_PATTERNS); + + this.taskPolicies = this.createPolicyStatements(); + } + + /** + * @internal + */ + protected _renderTask(): any { + return { + Resource: integrationResourceArn('emr-containers', 'deleteVirtualCluster', this.integrationPattern), + Parameters: sfn.FieldUtils.renderObject({ + Id: this.props.virtualClusterId.value, + }), + }; + }; + + private createPolicyStatements(): iam.PolicyStatement[] { + const actions = ['emr-containers:DeleteVirtualCluster']; + if (this.integrationPattern === sfn.IntegrationPattern.RUN_JOB) { + actions.push('emr-containers:DescribeVirtualCluster'); + } + + return [new iam.PolicyStatement({ + resources: [ + cdk.Stack.of(this).formatArn({ + arnFormat: cdk.ArnFormat.SLASH_RESOURCE_SLASH_RESOURCE_NAME, + service: 'emr-containers', + resource: 'virtualclusters', + resourceName: sfn.JsonPath.isEncodedJsonPath(this.props.virtualClusterId.value) ? '*' : this.props.virtualClusterId.value, + }), + ], + actions: actions, + })]; + } +} diff --git a/packages/@aws-cdk/aws-stepfunctions-tasks/lib/emrcontainers/start-job-run.ts b/packages/@aws-cdk/aws-stepfunctions-tasks/lib/emrcontainers/start-job-run.ts new file mode 100644 index 0000000000000..3b4289cc5345a --- /dev/null +++ b/packages/@aws-cdk/aws-stepfunctions-tasks/lib/emrcontainers/start-job-run.ts @@ -0,0 +1,681 @@ +import * as path from 'path'; +import * as iam from '@aws-cdk/aws-iam'; +import * as lambda from '@aws-cdk/aws-lambda'; +import * as logs from '@aws-cdk/aws-logs'; +import * as s3 from '@aws-cdk/aws-s3'; +import * as sfn from '@aws-cdk/aws-stepfunctions'; +import { TaskInput } from '@aws-cdk/aws-stepfunctions'; +import * as cdk from '@aws-cdk/core'; +import * as cr from '@aws-cdk/custom-resources'; +import * as awscli from '@aws-cdk/lambda-layer-awscli'; +import { Construct } from 'constructs'; +import { integrationResourceArn, validatePatternSupported } from '../private/task-utils'; + +/** + * The props for a EMR Containers StartJobRun Task. + */ +export interface EmrContainersStartJobRunProps extends sfn.TaskStateBaseProps { + /** + * The ID of the virtual cluster where the job will be run + */ + readonly virtualCluster: VirtualClusterInput; + + /** + * The name of the job run. + * + * @default - No job run name + */ + readonly jobName?: string; + + /** + * The execution role for the job run. + * + * If `virtualClusterId` is from a JSON input path, an execution role must be provided. + * If an execution role is provided, follow the documentation to update the role trust policy. + * @see https://docs.aws.amazon.com/emr/latest/EMR-on-EKS-DevelopmentGuide/setting-up-trust-policy.html + * + * @default - Automatically generated only when the provided `virtualClusterId` is not an encoded JSON path + */ + readonly executionRole?: iam.IRole; + + /** + * The Amazon EMR release version to use for the job run. + */ + readonly releaseLabel: ReleaseLabel; + + /** + * The configurations for the application running in the job run. + * + * Maximum of 100 items + * + * @see https://docs.aws.amazon.com/emr-on-eks/latest/APIReference/API_Configuration.html + * + * @default - No application config + */ + readonly applicationConfig?: ApplicationConfiguration[]; + + /** + * The job driver for the job run. + * + * @see https://docs.aws.amazon.com/emr-on-eks/latest/APIReference/API_JobDriver.html + */ + readonly jobDriver: JobDriver; + + /** + * Configuration for monitoring the job run + * + * @see https://docs.aws.amazon.com/emr-on-eks/latest/APIReference/API_MonitoringConfiguration.html + * + * @default - logging enabled and resources automatically generated if `monitoring.logging` is set to `true` + */ + readonly monitoring?: Monitoring; + + /** + * The tags assigned to job runs. + * + * @default - None + */ + readonly tags?: { [key: string]: string }; +} + +/** + * Starts a job run. + * + * A job is a unit of work that you submit to Amazon EMR on EKS for execution. + * The work performed by the job can be defined by a Spark jar, PySpark script, or SparkSQL query. + * A job run is an execution of the job on the virtual cluster. + * + * @see https://docs.aws.amazon.com/step-functions/latest/dg/connect-emr-eks.html + */ +export class EmrContainersStartJobRun extends sfn.TaskStateBase implements iam.IGrantable { + private static readonly SUPPORTED_INTEGRATION_PATTERNS: sfn.IntegrationPattern[] = [ + sfn.IntegrationPattern.REQUEST_RESPONSE, + sfn.IntegrationPattern.RUN_JOB, + ]; + + protected readonly taskMetrics?: sfn.TaskMetricsConfig; + protected readonly taskPolicies?: iam.PolicyStatement[]; + + public readonly grantPrincipal: iam.IPrincipal; + private role: iam.IRole; + private readonly logGroup?: logs.ILogGroup; + private readonly logBucket?: s3.IBucket; + private readonly integrationPattern: sfn.IntegrationPattern; + + constructor(scope: Construct, id: string, private readonly props: EmrContainersStartJobRunProps) { + super(scope, id, props); + this.integrationPattern = props.integrationPattern ?? sfn.IntegrationPattern.RUN_JOB; + validatePatternSupported(this.integrationPattern, EmrContainersStartJobRun.SUPPORTED_INTEGRATION_PATTERNS); + + if (this.props.applicationConfig) { + this.validateAppConfig(this.props.applicationConfig); + } + + if (this.props.jobDriver.sparkSubmitJobDriver) { + this.validateSparkSubmitJobDriver(props.jobDriver.sparkSubmitJobDriver); + } + + if (this.props.executionRole === undefined + && sfn.JsonPath.isEncodedJsonPath(props.virtualCluster.id)) { + throw new Error('Execution role cannot be undefined when the virtual cluster ID is not a concrete value. Provide an execution role with the correct trust policy'); + } + + this.logGroup = this.assignLogGroup(); + this.logBucket = this.assignLogBucket(); + this.role = this.props.executionRole ?? this.createJobExecutionRole(); + this.grantPrincipal = this.role; + + this.grantMonitoringPolicies(); + + this.taskPolicies = this.createPolicyStatements(); + } + + /** + * @internal + */ + protected _renderTask(): any { + return { + Resource: integrationResourceArn('emr-containers', 'startJobRun', this.integrationPattern), + Parameters: sfn.FieldUtils.renderObject({ + VirtualClusterId: this.props.virtualCluster.id, + Name: this.props.jobName, + ExecutionRoleArn: this.role.roleArn, + ReleaseLabel: this.props.releaseLabel.label, + JobDriver: { + SparkSubmitJobDriver: { + EntryPoint: this.props.jobDriver.sparkSubmitJobDriver?.entryPoint.value, + EntryPointArguments: this.props.jobDriver.sparkSubmitJobDriver?.entryPointArguments?.value, + SparkSubmitParameters: this.props.jobDriver.sparkSubmitJobDriver?.sparkSubmitParameters, + }, + }, + ConfigurationOverrides: { + ApplicationConfiguration: cdk.listMapper(this.applicationConfigPropertyToJson)(this.props.applicationConfig), + MonitoringConfiguration: { + CloudWatchMonitoringConfiguration: this.logGroup ? { + LogGroupName: this.logGroup.logGroupName, + LogStreamNamePrefix: this.props.monitoring!.logStreamNamePrefix, + } : undefined, + PersistentAppUI: (this.props.monitoring?.persistentAppUI === false) + ? 'DISABLED' + : 'ENABLED', + S3MonitoringConfiguration: this.logBucket ? { + LogUri: this.logBucket.s3UrlForObject(), + } : undefined, + }, + }, + Tags: this.props.tags, + }), + }; + } + + /** + * Render the EMR Containers ConfigurationProperty as JSON + */ + private applicationConfigPropertyToJson = (property: ApplicationConfiguration) => { + return { + Classification: cdk.stringToCloudFormation(property.classification.classificationStatement), + Properties: property.properties ? cdk.objectToCloudFormation(property.properties) : undefined, + Configurations: property.nestedConfig ? cdk.listMapper(this.applicationConfigPropertyToJson)(property.nestedConfig) : undefined, + }; + } + + private validateAppConfigPropertiesLength(appConfig: ApplicationConfiguration) { + if (appConfig?.properties === undefined) { + return; + } else if (Object.keys(appConfig.properties).length > 100) { + throw new Error(`Application configuration properties must have 100 or fewer entries. Received ${Object.keys(appConfig.properties).length}`); + } + } + + private validatePropertiesNestedAppConfigBothNotUndefined(appConfig: ApplicationConfiguration) { + if (appConfig?.properties === undefined && appConfig?.nestedConfig === undefined) { + throw new Error('Application configuration must have either properties or nested app configurations defined.'); + } + } + + private validateAppConfig(config?: ApplicationConfiguration[]) { + if (config === undefined) { + return; + } else if (config.length > 100) { + throw new Error(`Application configuration array must have 100 or fewer entries. Received ${config.length}`); + } else { + config.forEach(element => this.validateAppConfig(element.nestedConfig)); + config.forEach(element => this.validateAppConfigPropertiesLength(element)); + config.forEach(element => this.validatePropertiesNestedAppConfigBothNotUndefined(element)); + } + } + + private isArrayOfStrings(value: any): boolean { + return Array.isArray(value) && value.every(item => typeof item === 'string'); + } + + private validateEntryPointArguments (entryPointArguments:sfn.TaskInput) { + if (typeof entryPointArguments.value === 'string' && !sfn.JsonPath.isEncodedJsonPath(entryPointArguments.value)) { + throw new Error(`Entry point arguments must be a string array or encoded JSON path, but received a non JSON path string'); + .`); + } + if (!this.isArrayOfStrings(entryPointArguments.value)) { + throw new Error(`Entry point arguments must be a string array or encoded JSON path but received ${typeof entryPointArguments.value}.`); + } + } + + private validateEntryPointArgumentsLength (entryPointArguments:sfn.TaskInput) { + if (this.isArrayOfStrings(entryPointArguments.value) + && (entryPointArguments.value.length > 10280 || entryPointArguments.value.length < 1)) { + throw new Error(`Entry point arguments must be a string array between 1 and 10280 in length. Received ${entryPointArguments.value.length}.`); + } + } + + private validateSparkSubmitParametersLength (sparkSubmitParameters : string) { + if (sparkSubmitParameters.length > 102400 || sparkSubmitParameters.length < 1) { + throw new Error(`Spark submit parameters must be between 1 and 102400 characters in length. Received ${sparkSubmitParameters.length}.`); + } + } + private validateEntryPoint (entryPoint: TaskInput) { + if (!sfn.JsonPath.isEncodedJsonPath(entryPoint.value) && (entryPoint.value.length > 256|| entryPoint.value.length < 1)) { + throw new Error(`Entry point must be between 1 and 256 characters in length. Received ${entryPoint.value.length}.`); + } + } + + private validateSparkSubmitJobDriver (driver:SparkSubmitJobDriver) { + this.validateEntryPoint(driver.entryPoint); + if (driver.entryPointArguments) { + this.validateEntryPointArguments(driver.entryPointArguments); + this.validateEntryPointArgumentsLength(driver.entryPointArguments); + } + if (driver.sparkSubmitParameters) { + this.validateSparkSubmitParametersLength(driver.sparkSubmitParameters); + } + } + + private assignLogGroup = () : any => { + if (this.props.monitoring?.logGroup) { + return (this.props.monitoring?.logGroup); + } else { + return (this.props.monitoring?.logging ? new logs.LogGroup(this, 'Monitoring Log Group') : undefined); + } + } + + private assignLogBucket = () : any => { + if (this.props.monitoring?.logBucket) { + return (this.props.monitoring?.logBucket); + } else { + return (this.props.monitoring?.logging ? new s3.Bucket(this, 'Monitoring Bucket') : undefined); + } + } + + // https://docs.aws.amazon.com/emr/latest/EMR-on-EKS-DevelopmentGuide/creating-job-execution-role.html + private createJobExecutionRole(): iam.Role { + const jobExecutionRole = new iam.Role(this, 'Job-Execution-Role', { + assumedBy: new iam.CompositePrincipal( + new iam.ServicePrincipal('emr-containers.amazonaws.com'), + new iam.ServicePrincipal('states.amazonaws.com'), + ), + }); + + this.logBucket?.grantReadWrite(jobExecutionRole); + this.logGroup?.grantWrite(jobExecutionRole); + this.logGroup?.grant(jobExecutionRole, 'logs:DescribeLogStreams'); + + jobExecutionRole.addToPrincipalPolicy( + new iam.PolicyStatement({ + resources: [ + 'arn:aws:logs:*:*:*', + ], + actions: [ + 'logs:DescribeLogGroups', + ], + }), + ); + + this.updateRoleTrustPolicy(jobExecutionRole); + + return jobExecutionRole; + } + private grantMonitoringPolicies() { + + this.logBucket?.grantReadWrite(this.role); + this.logGroup?.grantWrite(this.role); + this.logGroup?.grant(this.role, 'logs:DescribeLogStreams'); + + this.role.addToPrincipalPolicy( + new iam.PolicyStatement({ + resources: [ + 'arn:aws:logs:*:*:*', + ], + actions: [ + 'logs:DescribeLogGroups', + ], + }), + ); + } + + /** + * If an execution role is not provided by user, the automatically generated job execution role must create a trust relationship + * between itself and the identity of the EMR managed service account in order to run jobs on the Kubernetes namespace. + * + * This cannot occur if the user provided virtualClusterId is within an encoded JSON path. + * + * The trust relationship can be created by updating the trust policy of the job execution role. + * + * @param role the automatically generated job execution role + */ + private updateRoleTrustPolicy(role: iam.Role) { + const eksClusterInfo = new cr.AwsCustomResource(this, 'GetEksClusterInfo', { + onCreate: { + service: 'EMRcontainers', + action: 'describeVirtualCluster', + parameters: { + id: this.props.virtualCluster.id, + }, + outputPaths: ['virtualCluster.containerProvider.info.eksInfo.namespace', 'virtualCluster.containerProvider.id'], + physicalResourceId: cr.PhysicalResourceId.of('id'), + }, + policy: cr.AwsCustomResourcePolicy.fromSdkCalls({ + resources: cr.AwsCustomResourcePolicy.ANY_RESOURCE, + }), + }); + /* We make use of custom resources to call update-roll-trust-policy as this command is only available through + * aws cli because this is only used during the initial setup and is not available through the sdk. + * https://awscli.amazonaws.com/v2/documentation/api/latest/reference/emr-containers/update-role-trust-policy.html + * Commands available through SDK: https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/EMRcontainers.html + * Commands available through CLI: https://awscli.amazonaws.com/v2/documentation/api/latest/reference/emr-containers/index.html + */ + const cliLayer = new awscli.AwsCliLayer(this, 'awsclilayer'); + const shellCliLambda = new lambda.SingletonFunction(this, 'Call Update-Role-Trust-Policy', { + uuid: '8693BB64-9689-44B6-9AAF-B0CC9EB8757C', + runtime: lambda.Runtime.PYTHON_3_6, + handler: 'index.handler', + code: lambda.Code.fromAsset(path.join(__dirname, 'utils/role-policy')), + timeout: cdk.Duration.seconds(30), + memorySize: 256, + layers: [cliLayer], + }); + shellCliLambda.addToRolePolicy( + new iam.PolicyStatement({ + resources: [ + cdk.Stack.of(this).formatArn({ + service: 'eks', + resource: 'cluster', + resourceName: eksClusterInfo.getResponseField('virtualCluster.containerProvider.id'), + }), + ], + actions: [ + 'eks:DescribeCluster', + ], + }), + ); + shellCliLambda.addToRolePolicy( + new iam.PolicyStatement({ + resources: [role.roleArn], + actions: [ + 'iam:GetRole', + 'iam:UpdateAssumeRolePolicy', + ], + }), + ); + const provider = new cr.Provider(this, 'CustomResourceProvider', { + onEventHandler: shellCliLambda, + }); + new cdk.CustomResource(this, 'Custom Resource', { + properties: { + eksNamespace: eksClusterInfo.getResponseField('virtualCluster.containerProvider.info.eksInfo.namespace'), + eksClusterId: eksClusterInfo.getResponseField('virtualCluster.containerProvider.id'), + roleName: role.roleName, + }, + serviceToken: provider.serviceToken, + }); + } + + private createPolicyStatements(): iam.PolicyStatement[] { + const policyStatements = [ + new iam.PolicyStatement({ + resources: [ + cdk.Stack.of(this).formatArn({ + arnFormat: cdk.ArnFormat.SLASH_RESOURCE_SLASH_RESOURCE_NAME, + service: 'emr-containers', + resource: 'virtualclusters', + resourceName: sfn.JsonPath.isEncodedJsonPath(this.props.virtualCluster.id) ? '*' : this.props.virtualCluster.id, // Need wild card for dynamic start job run https://docs.aws.amazon.com/step-functions/latest/dg/emr-eks-iam.html + }), + ], + actions: ['emr-containers:StartJobRun'], + conditions: { + StringEquals: { + 'emr-containers:ExecutionRoleArn': this.role.roleArn, + }, + }, + }), + ]; + + if (this.integrationPattern === sfn.IntegrationPattern.RUN_JOB) { + policyStatements.push( + new iam.PolicyStatement({ + resources: [ + cdk.Stack.of(this).formatArn({ + arnFormat: cdk.ArnFormat.SLASH_RESOURCE_SLASH_RESOURCE_NAME, + service: 'emr-containers', + resource: 'virtualclusters', + resourceName: sfn.JsonPath.isEncodedJsonPath(this.props.virtualCluster.id) ? '*' : `${this.props.virtualCluster.id}/jobruns/*`, // Need wild card for dynamic start job run https://docs.aws.amazon.com/step-functions/latest/dg/emr-eks-iam.html + }), + ], + actions: [ + 'emr-containers:DescribeJobRun', + 'emr-containers:CancelJobRun', + ], + }), + ); + } + + return policyStatements; + } +} + +/** + * The information about job driver for Spark submit. + */ +export interface SparkSubmitJobDriver { + /** + * The entry point of job application. + * + * Length Constraints: Minimum length of 1. Maximum length of 256. + */ + readonly entryPoint: sfn.TaskInput; + + /** + * The arguments for a job application in a task input object containing an array of strings + * + * Length Constraints: Minimum length of 1. Maximum length of 10280. + * @type sfn.TaskInput which expects payload as an array of strings + * + * @default - No arguments defined + */ + readonly entryPointArguments?: sfn.TaskInput; + + /** + * The Spark submit parameters that are used for job runs. + * + * Length Constraints: Minimum length of 1. Maximum length of 102400. + * + * @default - No spark submit parameters + */ + readonly sparkSubmitParameters?: string; +} + +/** + * Specify the driver that the EMR Containers job runs on. + * The job driver is used to provide an input for the job that will be run. + */ +export interface JobDriver { + /** + * The job driver parameters specified for spark submit. + * + * @see https://docs.aws.amazon.com/emr-on-eks/latest/APIReference/API_SparkSubmitJobDriver.html + * + */ + readonly sparkSubmitJobDriver: SparkSubmitJobDriver; +} + +/** + * The classification within a EMR Containers application configuration. + * Class can be extended to add other classifications. + * For example, new Classification('xxx-yyy'); + */ +export class Classification { + /** + * Sets the maximizeResourceAllocation property to true or false. + * When true, Amazon EMR automatically configures spark-defaults properties based on cluster hardware configuration. + * + * For more info: + * @see https://docs.aws.amazon.com/emr/latest/ReleaseGuide/emr-spark-configure.html#emr-spark-maximizeresourceallocation + */ + static readonly SPARK = new Classification('spark'); + + /** + * Sets values in the spark-defaults.conf file. + * + * For more info: + * @see https://spark.apache.org/docs/latest/configuration.html + */ + static readonly SPARK_DEFAULTS = new Classification('spark-defaults'); + + /** + * Sets values in the spark-env.sh file. + * + * For more info: + * @see https://spark.apache.org/docs/latest/configuration.html#environment-variables + */ + static readonly SPARK_ENV = new Classification('spark-env'); + + /** + * Sets values in the hive-site.xml for Spark. + */ + static readonly SPARK_HIVE_SITE = new Classification('spark-hive-site'); + + /** + * Sets values in the log4j.properties file. + * + * For more settings and info: + * @see https://github.com/apache/spark/blob/master/conf/log4j.properties.template + */ + static readonly SPARK_LOG4J = new Classification('spark-log4j'); + + /** + * Sets values in the metrics.properties file. + * + * For more settings and info: + * @see https://github.com/apache/spark/blob/master/conf/metrics.properties.template + */ + static readonly SPARK_METRICS = new Classification('spark-metrics'); + + /** + * Creates a new Classification + * + * @param classificationStatement A literal string in case a new EMR classification is released, if not already defined. + */ + constructor(public readonly classificationStatement: string) { } +} + +/** + * A configuration specification to be used when provisioning virtual clusters, + * which can include configurations for applications and software bundled with Amazon EMR on EKS. + * + * A configuration consists of a classification, properties, and optional nested configurations. + * A classification refers to an application-specific configuration file. + * Properties are the settings you want to change in that file. + * @see https://docs.aws.amazon.com/emr/latest/ReleaseGuide/emr-configure-apps.html + */ +export interface ApplicationConfiguration { + /** + * The classification within a configuration. + * + * Length Constraints: Minimum length of 1. Maximum length of 1024. + */ + readonly classification: Classification; + + /** + * A list of additional configurations to apply within a configuration object. + * + * Array Members: Maximum number of 100 items. + * + * @default - No other configurations + */ + readonly nestedConfig?: ApplicationConfiguration[]; + + /** + * A set of properties specified within a configuration classification. + * + * Map Entries: Maximum number of 100 items. + * + * @default - No properties + */ + readonly properties?: { [key: string]: string }; +} + +/** + * Configuration setting for monitoring. + */ +export interface Monitoring { + /** + * Enable logging for this job. + * + * If set to true, will automatically create a Cloudwatch Log Group and S3 bucket. + * This will be set to `true` implicitly if values are provided for `logGroup` or `logBucket`. + * + * @default true - true if values are provided for `logGroup` or `logBucket`, false otherwise + */ + readonly logging?: boolean + + /** + * A log group for CloudWatch monitoring. + * + * You can configure your jobs to send log information to CloudWatch Logs. + * + * @default - if `logging` is manually set to `true` and a `logGroup` is not provided, a `logGroup` will be automatically generated`. + */ + readonly logGroup?: logs.ILogGroup; + + /** + * A log stream name prefix for Cloudwatch monitoring. + * + * @default - Log streams created in this log group have no default prefix + */ + readonly logStreamNamePrefix?: string; + + /** + * Amazon S3 Bucket for monitoring log publishing. + * + * You can configure your jobs to send log information to Amazon S3. + * + * @default - if `logging` is manually set to `true` and a `logBucket` is not provided, a `logBucket` will be automatically generated`. + */ + readonly logBucket?: s3.IBucket; + + /** + * Monitoring configurations for the persistent application UI. + * + * @default true + */ + readonly persistentAppUI?: boolean; +} + +/** + * The Amazon EMR release version to use for the job run. + * + * Can be extended to include new EMR releases + * + * For example, `new ReleaseLabel('emr-x.xx.x-latest');` + */ +export class ReleaseLabel { + /** + * EMR Release version 5.32.0 + */ + static readonly EMR_5_32_0 = new ReleaseLabel('emr-5.32.0-latest'); + + /** + * EMR Release version 5.33.0 + */ + static readonly EMR_5_33_0 = new ReleaseLabel('emr-5.33.0-latest'); + + /** + * EMR Release version 6.2.0 + */ + static readonly EMR_6_2_0 = new ReleaseLabel('emr-6.2.0-latest'); + + /** + * EMR Release version 6.3.0 + */ + static readonly EMR_6_3_0 = new ReleaseLabel('emr-6.3.0-latest'); + + /** + * Initializes the label string. + * + * @param label A literal string that contains the release-version ex. 'emr-x.x.x-latest' + */ + constructor(public readonly label: string) { } +} + +/** + * Class that returns a virtual cluster's id depending on input type + */ +export class VirtualClusterInput { + /** + * Input for a virtualClusterId from a Task Input + */ + static fromTaskInput(taskInput: sfn.TaskInput): VirtualClusterInput { + return new VirtualClusterInput(taskInput.value); + } + + /** + * Input for virtualClusterId from a literal string + */ + static fromVirtualClusterId(virtualClusterId: string): VirtualClusterInput { + return new VirtualClusterInput(virtualClusterId); + } + + /** + * Initializes the virtual cluster ID. + * + * @param id The VirtualCluster Id + */ + private constructor(public readonly id: string) { } +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-stepfunctions-tasks/lib/emrcontainers/utils/role-policy/index.py b/packages/@aws-cdk/aws-stepfunctions-tasks/lib/emrcontainers/utils/role-policy/index.py new file mode 100644 index 0000000000000..c99c73c685d06 --- /dev/null +++ b/packages/@aws-cdk/aws-stepfunctions-tasks/lib/emrcontainers/utils/role-policy/index.py @@ -0,0 +1,16 @@ +import subprocess as sp +import os +import logging + +#https://github.com/aws/aws-cdk/tree/master/packages/%40aws-cdk/aws-stepfunctions#custom-state +def handler(event, context): + logger = logging.getLogger() + logger.setLevel(logging.INFO) + command = f"/opt/awscli/aws emr-containers update-role-trust-policy --cluster-name {event['ResourceProperties']['eksClusterId']} --namespace {event['ResourceProperties']['eksNamespace']} --role-name {event['ResourceProperties']['roleName']}" + if event['RequestType'] == 'Create' or event['RequestType'] == 'Update' : + try: + res = sp.check_output(command, shell=True) + logger.info(f"Successfully ran {command}") + except Exception as e: + logger.info(f"ERROR: {str(e)}") + \ No newline at end of file diff --git a/packages/@aws-cdk/aws-stepfunctions-tasks/lib/index.ts b/packages/@aws-cdk/aws-stepfunctions-tasks/lib/index.ts index 0c089eee35bae..04fd1846f1d08 100644 --- a/packages/@aws-cdk/aws-stepfunctions-tasks/lib/index.ts +++ b/packages/@aws-cdk/aws-stepfunctions-tasks/lib/index.ts @@ -29,6 +29,9 @@ export * from './emr/emr-add-step'; export * from './emr/emr-cancel-step'; export * from './emr/emr-modify-instance-fleet-by-name'; export * from './emr/emr-modify-instance-group-by-name'; +export * from './emrcontainers/create-virtual-cluster'; +export * from './emrcontainers/delete-virtual-cluster'; +export * from './emrcontainers/start-job-run'; export * from './glue/run-glue-job-task'; export * from './glue/start-job-run'; export * from './batch/run-batch-job'; diff --git a/packages/@aws-cdk/aws-stepfunctions-tasks/package.json b/packages/@aws-cdk/aws-stepfunctions-tasks/package.json index 09bfee9d4f4f0..a3b11eeb6b71b 100644 --- a/packages/@aws-cdk/aws-stepfunctions-tasks/package.json +++ b/packages/@aws-cdk/aws-stepfunctions-tasks/package.json @@ -107,11 +107,14 @@ "@aws-cdk/aws-iam": "0.0.0", "@aws-cdk/aws-kms": "0.0.0", "@aws-cdk/aws-lambda": "0.0.0", + "@aws-cdk/aws-logs": "0.0.0", "@aws-cdk/aws-s3": "0.0.0", "@aws-cdk/aws-sns": "0.0.0", "@aws-cdk/aws-sqs": "0.0.0", "@aws-cdk/aws-stepfunctions": "0.0.0", "@aws-cdk/core": "0.0.0", + "@aws-cdk/custom-resources": "0.0.0", + "@aws-cdk/lambda-layer-awscli": "0.0.0", "constructs": "^3.3.69" }, "homepage": "https://github.com/aws/aws-cdk", @@ -129,11 +132,14 @@ "@aws-cdk/aws-iam": "0.0.0", "@aws-cdk/aws-kms": "0.0.0", "@aws-cdk/aws-lambda": "0.0.0", + "@aws-cdk/aws-logs": "0.0.0", "@aws-cdk/aws-s3": "0.0.0", "@aws-cdk/aws-sns": "0.0.0", "@aws-cdk/aws-sqs": "0.0.0", "@aws-cdk/aws-stepfunctions": "0.0.0", "@aws-cdk/core": "0.0.0", + "@aws-cdk/custom-resources": "0.0.0", + "@aws-cdk/lambda-layer-awscli": "0.0.0", "constructs": "^3.3.69" }, "engines": { diff --git a/packages/@aws-cdk/aws-stepfunctions-tasks/test/emrcontainers/create-virtual-cluster.test.ts b/packages/@aws-cdk/aws-stepfunctions-tasks/test/emrcontainers/create-virtual-cluster.test.ts new file mode 100644 index 0000000000000..d7b6a3d925626 --- /dev/null +++ b/packages/@aws-cdk/aws-stepfunctions-tasks/test/emrcontainers/create-virtual-cluster.test.ts @@ -0,0 +1,215 @@ +import { Template } from '@aws-cdk/assertions'; +import * as eks from '@aws-cdk/aws-eks'; +import * as sfn from '@aws-cdk/aws-stepfunctions'; +import { Stack } from '@aws-cdk/core'; +import { EmrContainersCreateVirtualCluster, EksClusterInput } from '../../lib/emrcontainers/create-virtual-cluster'; + +const emrContainersVirtualClusterName = 'EMR Containers Virtual Cluster'; +let stack: Stack; +let clusterId: string; + +beforeEach(() => { + stack = new Stack(); + clusterId = 'test-eks'; +}); + +describe('Invoke emr-containers CreateVirtualCluster with ', () => { + test('only required properties', () => { + // WHEN + const task = new EmrContainersCreateVirtualCluster(stack, 'Task', { + eksCluster: EksClusterInput.fromTaskInput(sfn.TaskInput.fromText(clusterId)), + }); + + new sfn.StateMachine(stack, 'SM', { + definition: task, + }); + + // THEN + expect(stack.resolve(task.toStateJson())).toEqual({ + Type: 'Task', + Resource: { + 'Fn::Join': [ + '', + [ + 'arn:', + { + Ref: 'AWS::Partition', + }, + ':states:::emr-containers:createVirtualCluster', + ], + ], + }, + End: true, + Parameters: { + 'Name.$': "States.Format('{}/{}', $$.Execution.Name, $$.State.Name)", + 'ContainerProvider': { + Id: clusterId, + Info: { + EksInfo: { + Namespace: 'default', + }, + }, + Type: 'EKS', + }, + }, + }); + }); + + test('all required/non-required properties', () => { + // WHEN + const task = new EmrContainersCreateVirtualCluster(stack, 'Task', { + virtualClusterName: emrContainersVirtualClusterName, + eksCluster: EksClusterInput.fromTaskInput(sfn.TaskInput.fromText(clusterId)), + eksNamespace: 'namespace', + integrationPattern: sfn.IntegrationPattern.REQUEST_RESPONSE, + }); + + new sfn.StateMachine(stack, 'SM', { + definition: task, + }); + + // THEN + expect(stack.resolve(task.toStateJson())).toMatchObject({ + Parameters: { + Name: emrContainersVirtualClusterName, + ContainerProvider: { + Id: clusterId, + Info: { + EksInfo: { + Namespace: 'namespace', + }, + }, + }, + }, + }); + }); + + test('clusterId from payload', () => { + // WHEN + const task = new EmrContainersCreateVirtualCluster(stack, 'Task', { + eksCluster: EksClusterInput.fromTaskInput(sfn.TaskInput.fromJsonPathAt('$.ClusterId')), + }); + + // THEN + expect(stack.resolve(task.toStateJson())).toMatchObject({ + Parameters: { + ContainerProvider: { + 'Id.$': '$.ClusterId', + }, + }, + }); + }); + + test('with an existing EKS cluster', () => { + // WHEN + const eksCluster = new eks.Cluster(stack, 'EKS Cluster', { + version: eks.KubernetesVersion.V1_20, + }); + + const task = new EmrContainersCreateVirtualCluster(stack, 'Task', { + eksCluster: EksClusterInput.fromCluster(eksCluster), + }); + + // THEN + expect(stack.resolve(task.toStateJson())).toMatchObject({ + Parameters: { + ContainerProvider: { + Id: { + Ref: 'EKSClusterEDAD5FD1', + }, + }, + }, + }); + }); + + test('Tags', () => { + // WHEN + const task = new EmrContainersCreateVirtualCluster(stack, 'Task', { + eksCluster: EksClusterInput.fromTaskInput(sfn.TaskInput.fromText(clusterId)), + tags: { + key: 'value', + }, + }); + + new sfn.StateMachine(stack, 'SM', { + definition: task, + }); + + // THEN + expect(stack.resolve(task.toStateJson())).toMatchObject({ + Parameters: { + Tags: { + key: 'value', + }, + }, + }); + }); +}); + +test('Permitted role actions included for CreateVirtualCluster if service integration pattern is REQUEST_RESPONSE', () => { + // WHEN + const task = new EmrContainersCreateVirtualCluster(stack, 'Task', { + virtualClusterName: emrContainersVirtualClusterName, + eksCluster: EksClusterInput.fromTaskInput(sfn.TaskInput.fromText(clusterId)), + }); + + new sfn.StateMachine(stack, 'SM', { + definition: task, + }); + + // THEN + Template.fromStack(stack).hasResourceProperties('AWS::IAM::Policy', { + PolicyDocument: { + Statement: [{ + Action: 'emr-containers:CreateVirtualCluster', + Effect: 'Allow', + Resource: '*', + }, + { + Action: 'iam:CreateServiceLinkedRole', + Condition: { + StringLike: { + 'iam:AWSServiceName': 'emr-containers.amazonaws.com', + }, + }, + Effect: 'Allow', + Resource: { + 'Fn::Join': [ + '', + [ + 'arn:', + { + Ref: 'AWS::Partition', + }, + ':iam::', + { + Ref: 'AWS::AccountId', + }, + ':role/aws-service-role/emr-containers.amazonaws.com/AWSServiceRoleForAmazonEMRContainers', + ], + ], + }, + }], + }, + }); +}); + +test('Task throws if WAIT_FOR_TASK_TOKEN is supplied as service integration pattern', () => { + expect(() => { + new EmrContainersCreateVirtualCluster(stack, 'EMR Containers CreateVirtualCluster Job', { + virtualClusterName: emrContainersVirtualClusterName, + eksCluster: EksClusterInput.fromTaskInput(sfn.TaskInput.fromText(clusterId)), + integrationPattern: sfn.IntegrationPattern.WAIT_FOR_TASK_TOKEN, + }); + }).toThrow(/Unsupported service integration pattern. Supported Patterns: REQUEST_RESPONSE. Received: WAIT_FOR_TASK_TOKEN/); +}); + +test('Task throws if RUN_JOB is supplied as service integration pattern', () => { + expect(() => { + new EmrContainersCreateVirtualCluster(stack, 'EMR Containers CreateVirtualCluster Job', { + virtualClusterName: emrContainersVirtualClusterName, + eksCluster: EksClusterInput.fromTaskInput(sfn.TaskInput.fromText(clusterId)), + integrationPattern: sfn.IntegrationPattern.RUN_JOB, + }); + }).toThrow(/Unsupported service integration pattern. Supported Patterns: REQUEST_RESPONSE. Received: RUN_JOB/); +}); diff --git a/packages/@aws-cdk/aws-stepfunctions-tasks/test/emrcontainers/delete-virtual-cluster.test.ts b/packages/@aws-cdk/aws-stepfunctions-tasks/test/emrcontainers/delete-virtual-cluster.test.ts new file mode 100644 index 0000000000000..484e3f21c342e --- /dev/null +++ b/packages/@aws-cdk/aws-stepfunctions-tasks/test/emrcontainers/delete-virtual-cluster.test.ts @@ -0,0 +1,228 @@ +import { Template } from '@aws-cdk/assertions'; +import * as sfn from '@aws-cdk/aws-stepfunctions'; +import { Stack } from '@aws-cdk/core'; +import { EmrContainersDeleteVirtualCluster } from '../../lib/emrcontainers/delete-virtual-cluster'; + +let stack: Stack; +let virtualClusterId: string; + +beforeEach(() => { + stack = new Stack(); + virtualClusterId = 'x01f27i9a7cv1td52keaktr6j'; +}); + +describe('Invoke EMR Containers Delete Virtual cluster with ', () => { + test('a valid cluster ID', () => { + // WHEN + const task = new EmrContainersDeleteVirtualCluster(stack, 'Task', { + virtualClusterId: sfn.TaskInput.fromText(virtualClusterId), + }); + + // THEN + expect(stack.resolve(task.toStateJson())).toEqual({ + Type: 'Task', + Resource: { + 'Fn::Join': [ + '', + [ + 'arn:', + { + Ref: 'AWS::Partition', + }, + ':states:::emr-containers:deleteVirtualCluster', + ], + ], + }, + End: true, + Parameters: { + Id: virtualClusterId, + }, + }); + }); + + test('a RUN_JOB call', () => { + // WHEN + const task = new EmrContainersDeleteVirtualCluster(stack, 'Task', { + virtualClusterId: sfn.TaskInput.fromText(virtualClusterId), + integrationPattern: sfn.IntegrationPattern.RUN_JOB, + }); + + // THEN + expect(stack.resolve(task.toStateJson())).toEqual({ + Type: 'Task', + Resource: { + 'Fn::Join': [ + '', + [ + 'arn:', + { + Ref: 'AWS::Partition', + }, + ':states:::emr-containers:deleteVirtualCluster.sync', + ], + ], + }, + End: true, + Parameters: { + Id: virtualClusterId, + }, + }); + }); + + test('passing in JSON Path', () => { + // WHEN + const task = new EmrContainersDeleteVirtualCluster(stack, 'Task', { + virtualClusterId: sfn.TaskInput.fromJsonPathAt('$.VirtualClusterId'), + }); + + // THEN + expect(stack.resolve(task.toStateJson())).toMatchObject({ + Parameters: { + 'Id.$': '$.VirtualClusterId', + }, + }); + }); +}); + +describe('Valid policy statements and resources are passed ', () => { + test('to the state machine with a REQUEST_RESPONSE call', () => { + // WHEN + const task = new EmrContainersDeleteVirtualCluster(stack, 'Task', { + virtualClusterId: sfn.TaskInput.fromText(virtualClusterId), + }); + + new sfn.StateMachine(stack, 'SM', { + definition: task, + }); + + // THEN + Template.fromStack(stack).hasResourceProperties('AWS::IAM::Policy', { + PolicyDocument: { + Statement: [{ + Action: 'emr-containers:DeleteVirtualCluster', + Effect: 'Allow', + Resource: { + 'Fn::Join': [ + '', + [ + 'arn:', + { + Ref: 'AWS::Partition', + }, + ':emr-containers:', + { + Ref: 'AWS::Region', + }, + ':', + { + Ref: 'AWS::AccountId', + }, + `:/virtualclusters/${virtualClusterId}`, + ], + ], + }, + }], + }, + }); + }); + + test('to the state machine with a RUN_JOB call', () => { + // WHEN + const task = new EmrContainersDeleteVirtualCluster(stack, 'Task', { + virtualClusterId: sfn.TaskInput.fromText(virtualClusterId), + integrationPattern: sfn.IntegrationPattern.RUN_JOB, + }); + + new sfn.StateMachine(stack, 'SM', { + definition: task, + }); + + // THEN + Template.fromStack(stack).hasResourceProperties('AWS::IAM::Policy', { + PolicyDocument: { + Statement: [{ + Action: [ + 'emr-containers:DeleteVirtualCluster', + 'emr-containers:DescribeVirtualCluster', + ], + Effect: 'Allow', + Resource: { + 'Fn::Join': [ + '', + [ + 'arn:', + { + Ref: 'AWS::Partition', + }, + ':emr-containers:', + { + Ref: 'AWS::Region', + }, + ':', + { + Ref: 'AWS::AccountId', + }, + `:/virtualclusters/${virtualClusterId}`, + ], + ], + }, + }], + }, + }); + }); + + test('when the virtual cluster ID is from a payload', () => { + // WHEN + const task = new EmrContainersDeleteVirtualCluster(stack, 'Task', { + virtualClusterId: sfn.TaskInput.fromJsonPathAt('$.ClusterId'), + integrationPattern: sfn.IntegrationPattern.RUN_JOB, + }); + + new sfn.StateMachine(stack, 'SM', { + definition: task, + }); + + // THEN + Template.fromStack(stack).hasResourceProperties('AWS::IAM::Policy', { + PolicyDocument: { + Statement: [{ + Action: [ + 'emr-containers:DeleteVirtualCluster', + 'emr-containers:DescribeVirtualCluster', + ], + Effect: 'Allow', + Resource: { + 'Fn::Join': [ + '', + [ + 'arn:', + { + Ref: 'AWS::Partition', + }, + ':emr-containers:', + { + Ref: 'AWS::Region', + }, + ':', + { + Ref: 'AWS::AccountId', + }, + ':/virtualclusters/*', + ], + ], + }, + }], + }, + }); + }); +}); + + +test('Task throws if WAIT_FOR_TASK_TOKEN is supplied as service integration pattern', () => { + expect(() => { + new EmrContainersDeleteVirtualCluster(stack, 'EMR Containers DeleteVirtualCluster Job', { + virtualClusterId: sfn.TaskInput.fromText(virtualClusterId), + integrationPattern: sfn.IntegrationPattern.WAIT_FOR_TASK_TOKEN, + }); + }).toThrow(/Unsupported service integration pattern. Supported Patterns: REQUEST_RESPONSE,RUN_JOB. Received: WAIT_FOR_TASK_TOKEN/); +}); diff --git a/packages/@aws-cdk/aws-stepfunctions-tasks/test/emrcontainers/integ.job-submission-workflow.expected.json b/packages/@aws-cdk/aws-stepfunctions-tasks/test/emrcontainers/integ.job-submission-workflow.expected.json new file mode 100644 index 0000000000000..a3b99bf3fb6ed --- /dev/null +++ b/packages/@aws-cdk/aws-stepfunctions-tasks/test/emrcontainers/integ.job-submission-workflow.expected.json @@ -0,0 +1,1809 @@ +{ + "Resources": { + "integrationtesteksclusterDefaultVpc395E1A86": { + "Type": "AWS::EC2::VPC", + "Properties": { + "CidrBlock": "10.0.0.0/16", + "EnableDnsHostnames": true, + "EnableDnsSupport": true, + "InstanceTenancy": "default", + "Tags": [ + { + "Key": "Name", + "Value": "aws-stepfunctions-tasks-emr-containers-all-services-integ/integration-test-eks-cluster/DefaultVpc" + } + ] + } + }, + "integrationtesteksclusterDefaultVpcPublicSubnet1Subnet58061317": { + "Type": "AWS::EC2::Subnet", + "Properties": { + "CidrBlock": "10.0.0.0/19", + "VpcId": { + "Ref": "integrationtesteksclusterDefaultVpc395E1A86" + }, + "AvailabilityZone": "test-region-1a", + "MapPublicIpOnLaunch": true, + "Tags": [ + { + "Key": "aws-cdk:subnet-name", + "Value": "Public" + }, + { + "Key": "aws-cdk:subnet-type", + "Value": "Public" + }, + { + "Key": "kubernetes.io/role/elb", + "Value": "1" + }, + { + "Key": "Name", + "Value": "aws-stepfunctions-tasks-emr-containers-all-services-integ/integration-test-eks-cluster/DefaultVpc/PublicSubnet1" + } + ] + } + }, + "integrationtesteksclusterDefaultVpcPublicSubnet1RouteTable1D5A7569": { + "Type": "AWS::EC2::RouteTable", + "Properties": { + "VpcId": { + "Ref": "integrationtesteksclusterDefaultVpc395E1A86" + }, + "Tags": [ + { + "Key": "kubernetes.io/role/elb", + "Value": "1" + }, + { + "Key": "Name", + "Value": "aws-stepfunctions-tasks-emr-containers-all-services-integ/integration-test-eks-cluster/DefaultVpc/PublicSubnet1" + } + ] + } + }, + "integrationtesteksclusterDefaultVpcPublicSubnet1RouteTableAssociation4831B6A7": { + "Type": "AWS::EC2::SubnetRouteTableAssociation", + "Properties": { + "RouteTableId": { + "Ref": "integrationtesteksclusterDefaultVpcPublicSubnet1RouteTable1D5A7569" + }, + "SubnetId": { + "Ref": "integrationtesteksclusterDefaultVpcPublicSubnet1Subnet58061317" + } + } + }, + "integrationtesteksclusterDefaultVpcPublicSubnet1DefaultRoute33CE7FC3": { + "Type": "AWS::EC2::Route", + "Properties": { + "RouteTableId": { + "Ref": "integrationtesteksclusterDefaultVpcPublicSubnet1RouteTable1D5A7569" + }, + "DestinationCidrBlock": "0.0.0.0/0", + "GatewayId": { + "Ref": "integrationtesteksclusterDefaultVpcIGW9ADAFE6F" + } + }, + "DependsOn": [ + "integrationtesteksclusterDefaultVpcVPCGWE4DC2204" + ] + }, + "integrationtesteksclusterDefaultVpcPublicSubnet1EIP62A0A17B": { + "Type": "AWS::EC2::EIP", + "Properties": { + "Domain": "vpc", + "Tags": [ + { + "Key": "kubernetes.io/role/elb", + "Value": "1" + }, + { + "Key": "Name", + "Value": "aws-stepfunctions-tasks-emr-containers-all-services-integ/integration-test-eks-cluster/DefaultVpc/PublicSubnet1" + } + ] + } + }, + "integrationtesteksclusterDefaultVpcPublicSubnet1NATGatewayC9C984F9": { + "Type": "AWS::EC2::NatGateway", + "Properties": { + "SubnetId": { + "Ref": "integrationtesteksclusterDefaultVpcPublicSubnet1Subnet58061317" + }, + "AllocationId": { + "Fn::GetAtt": [ + "integrationtesteksclusterDefaultVpcPublicSubnet1EIP62A0A17B", + "AllocationId" + ] + }, + "Tags": [ + { + "Key": "kubernetes.io/role/elb", + "Value": "1" + }, + { + "Key": "Name", + "Value": "aws-stepfunctions-tasks-emr-containers-all-services-integ/integration-test-eks-cluster/DefaultVpc/PublicSubnet1" + } + ] + } + }, + "integrationtesteksclusterDefaultVpcPublicSubnet2Subnet68EAAF11": { + "Type": "AWS::EC2::Subnet", + "Properties": { + "CidrBlock": "10.0.32.0/19", + "VpcId": { + "Ref": "integrationtesteksclusterDefaultVpc395E1A86" + }, + "AvailabilityZone": "test-region-1b", + "MapPublicIpOnLaunch": true, + "Tags": [ + { + "Key": "aws-cdk:subnet-name", + "Value": "Public" + }, + { + "Key": "aws-cdk:subnet-type", + "Value": "Public" + }, + { + "Key": "kubernetes.io/role/elb", + "Value": "1" + }, + { + "Key": "Name", + "Value": "aws-stepfunctions-tasks-emr-containers-all-services-integ/integration-test-eks-cluster/DefaultVpc/PublicSubnet2" + } + ] + } + }, + "integrationtesteksclusterDefaultVpcPublicSubnet2RouteTableA4C7B327": { + "Type": "AWS::EC2::RouteTable", + "Properties": { + "VpcId": { + "Ref": "integrationtesteksclusterDefaultVpc395E1A86" + }, + "Tags": [ + { + "Key": "kubernetes.io/role/elb", + "Value": "1" + }, + { + "Key": "Name", + "Value": "aws-stepfunctions-tasks-emr-containers-all-services-integ/integration-test-eks-cluster/DefaultVpc/PublicSubnet2" + } + ] + } + }, + "integrationtesteksclusterDefaultVpcPublicSubnet2RouteTableAssociation62710C52": { + "Type": "AWS::EC2::SubnetRouteTableAssociation", + "Properties": { + "RouteTableId": { + "Ref": "integrationtesteksclusterDefaultVpcPublicSubnet2RouteTableA4C7B327" + }, + "SubnetId": { + "Ref": "integrationtesteksclusterDefaultVpcPublicSubnet2Subnet68EAAF11" + } + } + }, + "integrationtesteksclusterDefaultVpcPublicSubnet2DefaultRoute253A231E": { + "Type": "AWS::EC2::Route", + "Properties": { + "RouteTableId": { + "Ref": "integrationtesteksclusterDefaultVpcPublicSubnet2RouteTableA4C7B327" + }, + "DestinationCidrBlock": "0.0.0.0/0", + "GatewayId": { + "Ref": "integrationtesteksclusterDefaultVpcIGW9ADAFE6F" + } + }, + "DependsOn": [ + "integrationtesteksclusterDefaultVpcVPCGWE4DC2204" + ] + }, + "integrationtesteksclusterDefaultVpcPublicSubnet2EIPFC53AC43": { + "Type": "AWS::EC2::EIP", + "Properties": { + "Domain": "vpc", + "Tags": [ + { + "Key": "kubernetes.io/role/elb", + "Value": "1" + }, + { + "Key": "Name", + "Value": "aws-stepfunctions-tasks-emr-containers-all-services-integ/integration-test-eks-cluster/DefaultVpc/PublicSubnet2" + } + ] + } + }, + "integrationtesteksclusterDefaultVpcPublicSubnet2NATGatewayE109B761": { + "Type": "AWS::EC2::NatGateway", + "Properties": { + "SubnetId": { + "Ref": "integrationtesteksclusterDefaultVpcPublicSubnet2Subnet68EAAF11" + }, + "AllocationId": { + "Fn::GetAtt": [ + "integrationtesteksclusterDefaultVpcPublicSubnet2EIPFC53AC43", + "AllocationId" + ] + }, + "Tags": [ + { + "Key": "kubernetes.io/role/elb", + "Value": "1" + }, + { + "Key": "Name", + "Value": "aws-stepfunctions-tasks-emr-containers-all-services-integ/integration-test-eks-cluster/DefaultVpc/PublicSubnet2" + } + ] + } + }, + "integrationtesteksclusterDefaultVpcPublicSubnet3Subnet4307D52D": { + "Type": "AWS::EC2::Subnet", + "Properties": { + "CidrBlock": "10.0.64.0/19", + "VpcId": { + "Ref": "integrationtesteksclusterDefaultVpc395E1A86" + }, + "AvailabilityZone": "test-region-1c", + "MapPublicIpOnLaunch": true, + "Tags": [ + { + "Key": "aws-cdk:subnet-name", + "Value": "Public" + }, + { + "Key": "aws-cdk:subnet-type", + "Value": "Public" + }, + { + "Key": "kubernetes.io/role/elb", + "Value": "1" + }, + { + "Key": "Name", + "Value": "aws-stepfunctions-tasks-emr-containers-all-services-integ/integration-test-eks-cluster/DefaultVpc/PublicSubnet3" + } + ] + } + }, + "integrationtesteksclusterDefaultVpcPublicSubnet3RouteTable51E9DA7C": { + "Type": "AWS::EC2::RouteTable", + "Properties": { + "VpcId": { + "Ref": "integrationtesteksclusterDefaultVpc395E1A86" + }, + "Tags": [ + { + "Key": "kubernetes.io/role/elb", + "Value": "1" + }, + { + "Key": "Name", + "Value": "aws-stepfunctions-tasks-emr-containers-all-services-integ/integration-test-eks-cluster/DefaultVpc/PublicSubnet3" + } + ] + } + }, + "integrationtesteksclusterDefaultVpcPublicSubnet3RouteTableAssociation14A23BAF": { + "Type": "AWS::EC2::SubnetRouteTableAssociation", + "Properties": { + "RouteTableId": { + "Ref": "integrationtesteksclusterDefaultVpcPublicSubnet3RouteTable51E9DA7C" + }, + "SubnetId": { + "Ref": "integrationtesteksclusterDefaultVpcPublicSubnet3Subnet4307D52D" + } + } + }, + "integrationtesteksclusterDefaultVpcPublicSubnet3DefaultRouteB0117918": { + "Type": "AWS::EC2::Route", + "Properties": { + "RouteTableId": { + "Ref": "integrationtesteksclusterDefaultVpcPublicSubnet3RouteTable51E9DA7C" + }, + "DestinationCidrBlock": "0.0.0.0/0", + "GatewayId": { + "Ref": "integrationtesteksclusterDefaultVpcIGW9ADAFE6F" + } + }, + "DependsOn": [ + "integrationtesteksclusterDefaultVpcVPCGWE4DC2204" + ] + }, + "integrationtesteksclusterDefaultVpcPublicSubnet3EIP33828C59": { + "Type": "AWS::EC2::EIP", + "Properties": { + "Domain": "vpc", + "Tags": [ + { + "Key": "kubernetes.io/role/elb", + "Value": "1" + }, + { + "Key": "Name", + "Value": "aws-stepfunctions-tasks-emr-containers-all-services-integ/integration-test-eks-cluster/DefaultVpc/PublicSubnet3" + } + ] + } + }, + "integrationtesteksclusterDefaultVpcPublicSubnet3NATGateway01B2F0F1": { + "Type": "AWS::EC2::NatGateway", + "Properties": { + "SubnetId": { + "Ref": "integrationtesteksclusterDefaultVpcPublicSubnet3Subnet4307D52D" + }, + "AllocationId": { + "Fn::GetAtt": [ + "integrationtesteksclusterDefaultVpcPublicSubnet3EIP33828C59", + "AllocationId" + ] + }, + "Tags": [ + { + "Key": "kubernetes.io/role/elb", + "Value": "1" + }, + { + "Key": "Name", + "Value": "aws-stepfunctions-tasks-emr-containers-all-services-integ/integration-test-eks-cluster/DefaultVpc/PublicSubnet3" + } + ] + } + }, + "integrationtesteksclusterDefaultVpcPrivateSubnet1Subnet4E00CAFB": { + "Type": "AWS::EC2::Subnet", + "Properties": { + "CidrBlock": "10.0.96.0/19", + "VpcId": { + "Ref": "integrationtesteksclusterDefaultVpc395E1A86" + }, + "AvailabilityZone": "test-region-1a", + "MapPublicIpOnLaunch": false, + "Tags": [ + { + "Key": "aws-cdk:subnet-name", + "Value": "Private" + }, + { + "Key": "aws-cdk:subnet-type", + "Value": "Private" + }, + { + "Key": "kubernetes.io/role/internal-elb", + "Value": "1" + }, + { + "Key": "Name", + "Value": "aws-stepfunctions-tasks-emr-containers-all-services-integ/integration-test-eks-cluster/DefaultVpc/PrivateSubnet1" + } + ] + } + }, + "integrationtesteksclusterDefaultVpcPrivateSubnet1RouteTable4A47F4AC": { + "Type": "AWS::EC2::RouteTable", + "Properties": { + "VpcId": { + "Ref": "integrationtesteksclusterDefaultVpc395E1A86" + }, + "Tags": [ + { + "Key": "kubernetes.io/role/internal-elb", + "Value": "1" + }, + { + "Key": "Name", + "Value": "aws-stepfunctions-tasks-emr-containers-all-services-integ/integration-test-eks-cluster/DefaultVpc/PrivateSubnet1" + } + ] + } + }, + "integrationtesteksclusterDefaultVpcPrivateSubnet1RouteTableAssociation7482DD1E": { + "Type": "AWS::EC2::SubnetRouteTableAssociation", + "Properties": { + "RouteTableId": { + "Ref": "integrationtesteksclusterDefaultVpcPrivateSubnet1RouteTable4A47F4AC" + }, + "SubnetId": { + "Ref": "integrationtesteksclusterDefaultVpcPrivateSubnet1Subnet4E00CAFB" + } + } + }, + "integrationtesteksclusterDefaultVpcPrivateSubnet1DefaultRouteCC99A72C": { + "Type": "AWS::EC2::Route", + "Properties": { + "RouteTableId": { + "Ref": "integrationtesteksclusterDefaultVpcPrivateSubnet1RouteTable4A47F4AC" + }, + "DestinationCidrBlock": "0.0.0.0/0", + "NatGatewayId": { + "Ref": "integrationtesteksclusterDefaultVpcPublicSubnet1NATGatewayC9C984F9" + } + } + }, + "integrationtesteksclusterDefaultVpcPrivateSubnet2Subnet0C3539A8": { + "Type": "AWS::EC2::Subnet", + "Properties": { + "CidrBlock": "10.0.128.0/19", + "VpcId": { + "Ref": "integrationtesteksclusterDefaultVpc395E1A86" + }, + "AvailabilityZone": "test-region-1b", + "MapPublicIpOnLaunch": false, + "Tags": [ + { + "Key": "aws-cdk:subnet-name", + "Value": "Private" + }, + { + "Key": "aws-cdk:subnet-type", + "Value": "Private" + }, + { + "Key": "kubernetes.io/role/internal-elb", + "Value": "1" + }, + { + "Key": "Name", + "Value": "aws-stepfunctions-tasks-emr-containers-all-services-integ/integration-test-eks-cluster/DefaultVpc/PrivateSubnet2" + } + ] + } + }, + "integrationtesteksclusterDefaultVpcPrivateSubnet2RouteTableD7E59903": { + "Type": "AWS::EC2::RouteTable", + "Properties": { + "VpcId": { + "Ref": "integrationtesteksclusterDefaultVpc395E1A86" + }, + "Tags": [ + { + "Key": "kubernetes.io/role/internal-elb", + "Value": "1" + }, + { + "Key": "Name", + "Value": "aws-stepfunctions-tasks-emr-containers-all-services-integ/integration-test-eks-cluster/DefaultVpc/PrivateSubnet2" + } + ] + } + }, + "integrationtesteksclusterDefaultVpcPrivateSubnet2RouteTableAssociation99F934D5": { + "Type": "AWS::EC2::SubnetRouteTableAssociation", + "Properties": { + "RouteTableId": { + "Ref": "integrationtesteksclusterDefaultVpcPrivateSubnet2RouteTableD7E59903" + }, + "SubnetId": { + "Ref": "integrationtesteksclusterDefaultVpcPrivateSubnet2Subnet0C3539A8" + } + } + }, + "integrationtesteksclusterDefaultVpcPrivateSubnet2DefaultRoute50FF167F": { + "Type": "AWS::EC2::Route", + "Properties": { + "RouteTableId": { + "Ref": "integrationtesteksclusterDefaultVpcPrivateSubnet2RouteTableD7E59903" + }, + "DestinationCidrBlock": "0.0.0.0/0", + "NatGatewayId": { + "Ref": "integrationtesteksclusterDefaultVpcPublicSubnet2NATGatewayE109B761" + } + } + }, + "integrationtesteksclusterDefaultVpcPrivateSubnet3SubnetD17D4AF1": { + "Type": "AWS::EC2::Subnet", + "Properties": { + "CidrBlock": "10.0.160.0/19", + "VpcId": { + "Ref": "integrationtesteksclusterDefaultVpc395E1A86" + }, + "AvailabilityZone": "test-region-1c", + "MapPublicIpOnLaunch": false, + "Tags": [ + { + "Key": "aws-cdk:subnet-name", + "Value": "Private" + }, + { + "Key": "aws-cdk:subnet-type", + "Value": "Private" + }, + { + "Key": "kubernetes.io/role/internal-elb", + "Value": "1" + }, + { + "Key": "Name", + "Value": "aws-stepfunctions-tasks-emr-containers-all-services-integ/integration-test-eks-cluster/DefaultVpc/PrivateSubnet3" + } + ] + } + }, + "integrationtesteksclusterDefaultVpcPrivateSubnet3RouteTableA2B42791": { + "Type": "AWS::EC2::RouteTable", + "Properties": { + "VpcId": { + "Ref": "integrationtesteksclusterDefaultVpc395E1A86" + }, + "Tags": [ + { + "Key": "kubernetes.io/role/internal-elb", + "Value": "1" + }, + { + "Key": "Name", + "Value": "aws-stepfunctions-tasks-emr-containers-all-services-integ/integration-test-eks-cluster/DefaultVpc/PrivateSubnet3" + } + ] + } + }, + "integrationtesteksclusterDefaultVpcPrivateSubnet3RouteTableAssociationB70598F3": { + "Type": "AWS::EC2::SubnetRouteTableAssociation", + "Properties": { + "RouteTableId": { + "Ref": "integrationtesteksclusterDefaultVpcPrivateSubnet3RouteTableA2B42791" + }, + "SubnetId": { + "Ref": "integrationtesteksclusterDefaultVpcPrivateSubnet3SubnetD17D4AF1" + } + } + }, + "integrationtesteksclusterDefaultVpcPrivateSubnet3DefaultRoute6DE23E60": { + "Type": "AWS::EC2::Route", + "Properties": { + "RouteTableId": { + "Ref": "integrationtesteksclusterDefaultVpcPrivateSubnet3RouteTableA2B42791" + }, + "DestinationCidrBlock": "0.0.0.0/0", + "NatGatewayId": { + "Ref": "integrationtesteksclusterDefaultVpcPublicSubnet3NATGateway01B2F0F1" + } + } + }, + "integrationtesteksclusterDefaultVpcIGW9ADAFE6F": { + "Type": "AWS::EC2::InternetGateway", + "Properties": { + "Tags": [ + { + "Key": "Name", + "Value": "aws-stepfunctions-tasks-emr-containers-all-services-integ/integration-test-eks-cluster/DefaultVpc" + } + ] + } + }, + "integrationtesteksclusterDefaultVpcVPCGWE4DC2204": { + "Type": "AWS::EC2::VPCGatewayAttachment", + "Properties": { + "VpcId": { + "Ref": "integrationtesteksclusterDefaultVpc395E1A86" + }, + "InternetGatewayId": { + "Ref": "integrationtesteksclusterDefaultVpcIGW9ADAFE6F" + } + } + }, + "integrationtesteksclusterRole03F70AF0": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "eks.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/AmazonEKSClusterPolicy" + ] + ] + } + ] + } + }, + "integrationtesteksclusterControlPlaneSecurityGroup6E92F333": { + "Type": "AWS::EC2::SecurityGroup", + "Properties": { + "GroupDescription": "EKS Control Plane Security Group", + "SecurityGroupEgress": [ + { + "CidrIp": "0.0.0.0/0", + "Description": "Allow all outbound traffic by default", + "IpProtocol": "-1" + } + ], + "VpcId": { + "Ref": "integrationtesteksclusterDefaultVpc395E1A86" + } + } + }, + "integrationtesteksclusterCreationRoleB98FE02A": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "AWS": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::", + { + "Ref": "AWS::AccountId" + }, + ":root" + ] + ] + } + } + } + ], + "Version": "2012-10-17" + } + }, + "DependsOn": [ + "integrationtesteksclusterDefaultVpcIGW9ADAFE6F", + "integrationtesteksclusterDefaultVpcPrivateSubnet1DefaultRouteCC99A72C", + "integrationtesteksclusterDefaultVpcPrivateSubnet1RouteTable4A47F4AC", + "integrationtesteksclusterDefaultVpcPrivateSubnet1RouteTableAssociation7482DD1E", + "integrationtesteksclusterDefaultVpcPrivateSubnet1Subnet4E00CAFB", + "integrationtesteksclusterDefaultVpcPrivateSubnet2DefaultRoute50FF167F", + "integrationtesteksclusterDefaultVpcPrivateSubnet2RouteTableD7E59903", + "integrationtesteksclusterDefaultVpcPrivateSubnet2RouteTableAssociation99F934D5", + "integrationtesteksclusterDefaultVpcPrivateSubnet2Subnet0C3539A8", + "integrationtesteksclusterDefaultVpcPrivateSubnet3DefaultRoute6DE23E60", + "integrationtesteksclusterDefaultVpcPrivateSubnet3RouteTableA2B42791", + "integrationtesteksclusterDefaultVpcPrivateSubnet3RouteTableAssociationB70598F3", + "integrationtesteksclusterDefaultVpcPrivateSubnet3SubnetD17D4AF1", + "integrationtesteksclusterDefaultVpcPublicSubnet1DefaultRoute33CE7FC3", + "integrationtesteksclusterDefaultVpcPublicSubnet1EIP62A0A17B", + "integrationtesteksclusterDefaultVpcPublicSubnet1NATGatewayC9C984F9", + "integrationtesteksclusterDefaultVpcPublicSubnet1RouteTable1D5A7569", + "integrationtesteksclusterDefaultVpcPublicSubnet1RouteTableAssociation4831B6A7", + "integrationtesteksclusterDefaultVpcPublicSubnet1Subnet58061317", + "integrationtesteksclusterDefaultVpcPublicSubnet2DefaultRoute253A231E", + "integrationtesteksclusterDefaultVpcPublicSubnet2EIPFC53AC43", + "integrationtesteksclusterDefaultVpcPublicSubnet2NATGatewayE109B761", + "integrationtesteksclusterDefaultVpcPublicSubnet2RouteTableA4C7B327", + "integrationtesteksclusterDefaultVpcPublicSubnet2RouteTableAssociation62710C52", + "integrationtesteksclusterDefaultVpcPublicSubnet2Subnet68EAAF11", + "integrationtesteksclusterDefaultVpcPublicSubnet3DefaultRouteB0117918", + "integrationtesteksclusterDefaultVpcPublicSubnet3EIP33828C59", + "integrationtesteksclusterDefaultVpcPublicSubnet3NATGateway01B2F0F1", + "integrationtesteksclusterDefaultVpcPublicSubnet3RouteTable51E9DA7C", + "integrationtesteksclusterDefaultVpcPublicSubnet3RouteTableAssociation14A23BAF", + "integrationtesteksclusterDefaultVpcPublicSubnet3Subnet4307D52D", + "integrationtesteksclusterDefaultVpc395E1A86", + "integrationtesteksclusterDefaultVpcVPCGWE4DC2204" + ] + }, + "integrationtesteksclusterCreationRoleDefaultPolicy5417802D": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": "iam:PassRole", + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "integrationtesteksclusterRole03F70AF0", + "Arn" + ] + } + }, + { + "Action": [ + "eks:CreateCluster", + "eks:DescribeCluster", + "eks:DescribeUpdate", + "eks:DeleteCluster", + "eks:UpdateClusterVersion", + "eks:UpdateClusterConfig", + "eks:CreateFargateProfile", + "eks:TagResource", + "eks:UntagResource" + ], + "Effect": "Allow", + "Resource": [ + "*" + ] + }, + { + "Action": [ + "eks:DescribeFargateProfile", + "eks:DeleteFargateProfile" + ], + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": [ + "iam:GetRole", + "iam:listAttachedRolePolicies" + ], + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": "iam:CreateServiceLinkedRole", + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": [ + "ec2:DescribeInstances", + "ec2:DescribeNetworkInterfaces", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:DescribeRouteTables", + "ec2:DescribeDhcpOptions", + "ec2:DescribeVpcs" + ], + "Effect": "Allow", + "Resource": "*" + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "integrationtesteksclusterCreationRoleDefaultPolicy5417802D", + "Roles": [ + { + "Ref": "integrationtesteksclusterCreationRoleB98FE02A" + } + ] + }, + "DependsOn": [ + "integrationtesteksclusterDefaultVpcIGW9ADAFE6F", + "integrationtesteksclusterDefaultVpcPrivateSubnet1DefaultRouteCC99A72C", + "integrationtesteksclusterDefaultVpcPrivateSubnet1RouteTable4A47F4AC", + "integrationtesteksclusterDefaultVpcPrivateSubnet1RouteTableAssociation7482DD1E", + "integrationtesteksclusterDefaultVpcPrivateSubnet1Subnet4E00CAFB", + "integrationtesteksclusterDefaultVpcPrivateSubnet2DefaultRoute50FF167F", + "integrationtesteksclusterDefaultVpcPrivateSubnet2RouteTableD7E59903", + "integrationtesteksclusterDefaultVpcPrivateSubnet2RouteTableAssociation99F934D5", + "integrationtesteksclusterDefaultVpcPrivateSubnet2Subnet0C3539A8", + "integrationtesteksclusterDefaultVpcPrivateSubnet3DefaultRoute6DE23E60", + "integrationtesteksclusterDefaultVpcPrivateSubnet3RouteTableA2B42791", + "integrationtesteksclusterDefaultVpcPrivateSubnet3RouteTableAssociationB70598F3", + "integrationtesteksclusterDefaultVpcPrivateSubnet3SubnetD17D4AF1", + "integrationtesteksclusterDefaultVpcPublicSubnet1DefaultRoute33CE7FC3", + "integrationtesteksclusterDefaultVpcPublicSubnet1EIP62A0A17B", + "integrationtesteksclusterDefaultVpcPublicSubnet1NATGatewayC9C984F9", + "integrationtesteksclusterDefaultVpcPublicSubnet1RouteTable1D5A7569", + "integrationtesteksclusterDefaultVpcPublicSubnet1RouteTableAssociation4831B6A7", + "integrationtesteksclusterDefaultVpcPublicSubnet1Subnet58061317", + "integrationtesteksclusterDefaultVpcPublicSubnet2DefaultRoute253A231E", + "integrationtesteksclusterDefaultVpcPublicSubnet2EIPFC53AC43", + "integrationtesteksclusterDefaultVpcPublicSubnet2NATGatewayE109B761", + "integrationtesteksclusterDefaultVpcPublicSubnet2RouteTableA4C7B327", + "integrationtesteksclusterDefaultVpcPublicSubnet2RouteTableAssociation62710C52", + "integrationtesteksclusterDefaultVpcPublicSubnet2Subnet68EAAF11", + "integrationtesteksclusterDefaultVpcPublicSubnet3DefaultRouteB0117918", + "integrationtesteksclusterDefaultVpcPublicSubnet3EIP33828C59", + "integrationtesteksclusterDefaultVpcPublicSubnet3NATGateway01B2F0F1", + "integrationtesteksclusterDefaultVpcPublicSubnet3RouteTable51E9DA7C", + "integrationtesteksclusterDefaultVpcPublicSubnet3RouteTableAssociation14A23BAF", + "integrationtesteksclusterDefaultVpcPublicSubnet3Subnet4307D52D", + "integrationtesteksclusterDefaultVpc395E1A86", + "integrationtesteksclusterDefaultVpcVPCGWE4DC2204" + ] + }, + "integrationtesteksclusterE5C0ED98": { + "Type": "Custom::AWSCDK-EKS-Cluster", + "Properties": { + "ServiceToken": { + "Fn::GetAtt": [ + "awscdkawseksClusterResourceProviderNestedStackawscdkawseksClusterResourceProviderNestedStackResource9827C454", + "Outputs.awsstepfunctionstasksemrcontainersallservicesintegawscdkawseksClusterResourceProviderframeworkonEventFF3F425BArn" + ] + }, + "Config": { + "version": "1.21", + "roleArn": { + "Fn::GetAtt": [ + "integrationtesteksclusterRole03F70AF0", + "Arn" + ] + }, + "resourcesVpcConfig": { + "subnetIds": [ + { + "Ref": "integrationtesteksclusterDefaultVpcPublicSubnet1Subnet58061317" + }, + { + "Ref": "integrationtesteksclusterDefaultVpcPublicSubnet2Subnet68EAAF11" + }, + { + "Ref": "integrationtesteksclusterDefaultVpcPublicSubnet3Subnet4307D52D" + }, + { + "Ref": "integrationtesteksclusterDefaultVpcPrivateSubnet1Subnet4E00CAFB" + }, + { + "Ref": "integrationtesteksclusterDefaultVpcPrivateSubnet2Subnet0C3539A8" + }, + { + "Ref": "integrationtesteksclusterDefaultVpcPrivateSubnet3SubnetD17D4AF1" + } + ], + "securityGroupIds": [ + { + "Fn::GetAtt": [ + "integrationtesteksclusterControlPlaneSecurityGroup6E92F333", + "GroupId" + ] + } + ], + "endpointPublicAccess": true, + "endpointPrivateAccess": true + } + }, + "AssumeRoleArn": { + "Fn::GetAtt": [ + "integrationtesteksclusterCreationRoleB98FE02A", + "Arn" + ] + }, + "AttributesRevision": 2 + }, + "DependsOn": [ + "integrationtesteksclusterDefaultVpcIGW9ADAFE6F", + "integrationtesteksclusterDefaultVpcPrivateSubnet1DefaultRouteCC99A72C", + "integrationtesteksclusterDefaultVpcPrivateSubnet1RouteTable4A47F4AC", + "integrationtesteksclusterDefaultVpcPrivateSubnet1RouteTableAssociation7482DD1E", + "integrationtesteksclusterDefaultVpcPrivateSubnet1Subnet4E00CAFB", + "integrationtesteksclusterDefaultVpcPrivateSubnet2DefaultRoute50FF167F", + "integrationtesteksclusterDefaultVpcPrivateSubnet2RouteTableD7E59903", + "integrationtesteksclusterDefaultVpcPrivateSubnet2RouteTableAssociation99F934D5", + "integrationtesteksclusterDefaultVpcPrivateSubnet2Subnet0C3539A8", + "integrationtesteksclusterDefaultVpcPrivateSubnet3DefaultRoute6DE23E60", + "integrationtesteksclusterDefaultVpcPrivateSubnet3RouteTableA2B42791", + "integrationtesteksclusterDefaultVpcPrivateSubnet3RouteTableAssociationB70598F3", + "integrationtesteksclusterDefaultVpcPrivateSubnet3SubnetD17D4AF1", + "integrationtesteksclusterDefaultVpcPublicSubnet1DefaultRoute33CE7FC3", + "integrationtesteksclusterDefaultVpcPublicSubnet1EIP62A0A17B", + "integrationtesteksclusterDefaultVpcPublicSubnet1NATGatewayC9C984F9", + "integrationtesteksclusterDefaultVpcPublicSubnet1RouteTable1D5A7569", + "integrationtesteksclusterDefaultVpcPublicSubnet1RouteTableAssociation4831B6A7", + "integrationtesteksclusterDefaultVpcPublicSubnet1Subnet58061317", + "integrationtesteksclusterDefaultVpcPublicSubnet2DefaultRoute253A231E", + "integrationtesteksclusterDefaultVpcPublicSubnet2EIPFC53AC43", + "integrationtesteksclusterDefaultVpcPublicSubnet2NATGatewayE109B761", + "integrationtesteksclusterDefaultVpcPublicSubnet2RouteTableA4C7B327", + "integrationtesteksclusterDefaultVpcPublicSubnet2RouteTableAssociation62710C52", + "integrationtesteksclusterDefaultVpcPublicSubnet2Subnet68EAAF11", + "integrationtesteksclusterDefaultVpcPublicSubnet3DefaultRouteB0117918", + "integrationtesteksclusterDefaultVpcPublicSubnet3EIP33828C59", + "integrationtesteksclusterDefaultVpcPublicSubnet3NATGateway01B2F0F1", + "integrationtesteksclusterDefaultVpcPublicSubnet3RouteTable51E9DA7C", + "integrationtesteksclusterDefaultVpcPublicSubnet3RouteTableAssociation14A23BAF", + "integrationtesteksclusterDefaultVpcPublicSubnet3Subnet4307D52D", + "integrationtesteksclusterDefaultVpc395E1A86", + "integrationtesteksclusterDefaultVpcVPCGWE4DC2204", + "integrationtesteksclusterCreationRoleDefaultPolicy5417802D", + "integrationtesteksclusterCreationRoleB98FE02A" + ], + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete" + }, + "integrationtesteksclusterKubectlReadyBarrier0D4A21B0": { + "Type": "AWS::SSM::Parameter", + "Properties": { + "Type": "String", + "Value": "aws:cdk:eks:kubectl-ready" + }, + "DependsOn": [ + "integrationtesteksclusterCreationRoleDefaultPolicy5417802D", + "integrationtesteksclusterCreationRoleB98FE02A", + "integrationtesteksclusterE5C0ED98" + ] + }, + "integrationtesteksclusterMastersRole63B9B0BF": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "AWS": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::", + { + "Ref": "AWS::AccountId" + }, + ":root" + ] + ] + } + } + } + ], + "Version": "2012-10-17" + } + } + }, + "integrationtesteksclusterAwsAuthmanifestAEF9C6DF": { + "Type": "Custom::AWSCDK-EKS-KubernetesResource", + "Properties": { + "ServiceToken": { + "Fn::GetAtt": [ + "awscdkawseksKubectlProviderNestedStackawscdkawseksKubectlProviderNestedStackResourceA7AEBA6B", + "Outputs.awsstepfunctionstasksemrcontainersallservicesintegawscdkawseksKubectlProviderframeworkonEvent3B33A326Arn" + ] + }, + "Manifest": { + "Fn::Join": [ + "", + [ + "[{\"apiVersion\":\"v1\",\"kind\":\"ConfigMap\",\"metadata\":{\"name\":\"aws-auth\",\"namespace\":\"kube-system\",\"labels\":{\"aws.cdk.eks/prune-c89f1e1be2a935f1b46af591dd13f7d1a5d084570d\":\"\"}},\"data\":{\"mapRoles\":\"[{\\\"rolearn\\\":\\\"", + { + "Fn::GetAtt": [ + "integrationtesteksclusterMastersRole63B9B0BF", + "Arn" + ] + }, + "\\\",\\\"username\\\":\\\"", + { + "Fn::GetAtt": [ + "integrationtesteksclusterMastersRole63B9B0BF", + "Arn" + ] + }, + "\\\",\\\"groups\\\":[\\\"system:masters\\\"]},{\\\"rolearn\\\":\\\"", + { + "Fn::GetAtt": [ + "integrationtesteksclusterNodegroupDefaultCapacityNodeGroupRole75D45BA7", + "Arn" + ] + }, + "\\\",\\\"username\\\":\\\"system:node:{{EC2PrivateDNSName}}\\\",\\\"groups\\\":[\\\"system:bootstrappers\\\",\\\"system:nodes\\\"]}]\",\"mapUsers\":\"[]\",\"mapAccounts\":\"[]\"}}]" + ] + ] + }, + "ClusterName": { + "Ref": "integrationtesteksclusterE5C0ED98" + }, + "RoleArn": { + "Fn::GetAtt": [ + "integrationtesteksclusterCreationRoleB98FE02A", + "Arn" + ] + }, + "PruneLabel": "aws.cdk.eks/prune-c89f1e1be2a935f1b46af591dd13f7d1a5d084570d", + "Overwrite": true + }, + "DependsOn": [ + "integrationtesteksclusterKubectlReadyBarrier0D4A21B0" + ], + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete" + }, + "integrationtesteksclusterNodegroupDefaultCapacityNodeGroupRole75D45BA7": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": { + "Fn::Join": [ + "", + [ + "ec2.", + { + "Ref": "AWS::URLSuffix" + } + ] + ] + } + } + } + ], + "Version": "2012-10-17" + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/AmazonEKSWorkerNodePolicy" + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/AmazonEKS_CNI_Policy" + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" + ] + ] + } + ] + } + }, + "integrationtesteksclusterNodegroupDefaultCapacity536CF32C": { + "Type": "AWS::EKS::Nodegroup", + "Properties": { + "ClusterName": { + "Ref": "integrationtesteksclusterE5C0ED98" + }, + "NodeRole": { + "Fn::GetAtt": [ + "integrationtesteksclusterNodegroupDefaultCapacityNodeGroupRole75D45BA7", + "Arn" + ] + }, + "Subnets": [ + { + "Ref": "integrationtesteksclusterDefaultVpcPrivateSubnet1Subnet4E00CAFB" + }, + { + "Ref": "integrationtesteksclusterDefaultVpcPrivateSubnet2Subnet0C3539A8" + }, + { + "Ref": "integrationtesteksclusterDefaultVpcPrivateSubnet3SubnetD17D4AF1" + } + ], + "AmiType": "AL2_x86_64", + "ForceUpdateEnabled": true, + "InstanceTypes": [ + "m5.xlarge" + ], + "ScalingConfig": { + "DesiredSize": 3, + "MaxSize": 3, + "MinSize": 3 + } + } + }, + "awscdkawseksClusterResourceProviderNestedStackawscdkawseksClusterResourceProviderNestedStackResource9827C454": { + "Type": "AWS::CloudFormation::Stack", + "Properties": { + "TemplateURL": { + "Fn::Join": [ + "", + [ + "https://s3.", + { + "Ref": "AWS::Region" + }, + ".", + { + "Ref": "AWS::URLSuffix" + }, + "/", + { + "Ref": "AssetParameters8d0ccbee43ce642958da8926fed939a9fde9d6631e9297740d640c0859ca5945S3BucketC636D050" + }, + "/", + { + "Fn::Select": [ + 0, + { + "Fn::Split": [ + "||", + { + "Ref": "AssetParameters8d0ccbee43ce642958da8926fed939a9fde9d6631e9297740d640c0859ca5945S3VersionKey40C5ADFE" + } + ] + } + ] + }, + { + "Fn::Select": [ + 1, + { + "Fn::Split": [ + "||", + { + "Ref": "AssetParameters8d0ccbee43ce642958da8926fed939a9fde9d6631e9297740d640c0859ca5945S3VersionKey40C5ADFE" + } + ] + } + ] + } + ] + ] + }, + "Parameters": { + "referencetoawsstepfunctionstasksemrcontainersallservicesintegAssetParameters26ac61b4195cccf80ff73f332788ad7ffaab36d81ce570340a583a8364901665S3Bucket780F031ARef": { + "Ref": "AssetParameters26ac61b4195cccf80ff73f332788ad7ffaab36d81ce570340a583a8364901665S3Bucket1B280681" + }, + "referencetoawsstepfunctionstasksemrcontainersallservicesintegAssetParameters26ac61b4195cccf80ff73f332788ad7ffaab36d81ce570340a583a8364901665S3VersionKey8DCA7AE1Ref": { + "Ref": "AssetParameters26ac61b4195cccf80ff73f332788ad7ffaab36d81ce570340a583a8364901665S3VersionKeyB1E02791" + }, + "referencetoawsstepfunctionstasksemrcontainersallservicesintegintegrationtesteksclusterCreationRole78F8A91EArn": { + "Fn::GetAtt": [ + "integrationtesteksclusterCreationRoleB98FE02A", + "Arn" + ] + }, + "referencetoawsstepfunctionstasksemrcontainersallservicesintegAssetParameters5afea6e8e6c743a8d1766f21465e28d471e56bcb95c5970054b0514bc62a3720S3Bucket42FF7F94Ref": { + "Ref": "AssetParameters5afea6e8e6c743a8d1766f21465e28d471e56bcb95c5970054b0514bc62a3720S3Bucket3B443230" + }, + "referencetoawsstepfunctionstasksemrcontainersallservicesintegAssetParameters5afea6e8e6c743a8d1766f21465e28d471e56bcb95c5970054b0514bc62a3720S3VersionKeyD44C8135Ref": { + "Ref": "AssetParameters5afea6e8e6c743a8d1766f21465e28d471e56bcb95c5970054b0514bc62a3720S3VersionKeyAA4674FB" + }, + "referencetoawsstepfunctionstasksemrcontainersallservicesintegAssetParametersdaeb79e3cee39c9b902dc0d5c780223e227ed573ea60976252947adab5fb2be1S3Bucket5978B8B5Ref": { + "Ref": "AssetParametersdaeb79e3cee39c9b902dc0d5c780223e227ed573ea60976252947adab5fb2be1S3BucketDC4B98B1" + }, + "referencetoawsstepfunctionstasksemrcontainersallservicesintegAssetParametersdaeb79e3cee39c9b902dc0d5c780223e227ed573ea60976252947adab5fb2be1S3VersionKey9D9F3D17Ref": { + "Ref": "AssetParametersdaeb79e3cee39c9b902dc0d5c780223e227ed573ea60976252947adab5fb2be1S3VersionKeyA495226F" + } + } + }, + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete" + }, + "awscdkawseksKubectlProviderNestedStackawscdkawseksKubectlProviderNestedStackResourceA7AEBA6B": { + "Type": "AWS::CloudFormation::Stack", + "Properties": { + "TemplateURL": { + "Fn::Join": [ + "", + [ + "https://s3.", + { + "Ref": "AWS::Region" + }, + ".", + { + "Ref": "AWS::URLSuffix" + }, + "/", + { + "Ref": "AssetParameters249207c004483eea61658aceb796afc5f7a8b39d3b2333951c04ac8787af6d50S3BucketDBE4A868" + }, + "/", + { + "Fn::Select": [ + 0, + { + "Fn::Split": [ + "||", + { + "Ref": "AssetParameters249207c004483eea61658aceb796afc5f7a8b39d3b2333951c04ac8787af6d50S3VersionKeyDA4E6828" + } + ] + } + ] + }, + { + "Fn::Select": [ + 1, + { + "Fn::Split": [ + "||", + { + "Ref": "AssetParameters249207c004483eea61658aceb796afc5f7a8b39d3b2333951c04ac8787af6d50S3VersionKeyDA4E6828" + } + ] + } + ] + } + ] + ] + }, + "Parameters": { + "referencetoawsstepfunctionstasksemrcontainersallservicesintegintegrationtestekscluster4FFBB19EArn": { + "Fn::GetAtt": [ + "integrationtesteksclusterE5C0ED98", + "Arn" + ] + }, + "referencetoawsstepfunctionstasksemrcontainersallservicesintegintegrationtesteksclusterCreationRole78F8A91EArn": { + "Fn::GetAtt": [ + "integrationtesteksclusterCreationRoleB98FE02A", + "Arn" + ] + }, + "referencetoawsstepfunctionstasksemrcontainersallservicesintegAssetParameters4129bbca38164ecb28fee8e5b674f0d05e5957b4b8ed97d9c950527b5cc4ce10S3BucketFB7BE533Ref": { + "Ref": "AssetParameters4129bbca38164ecb28fee8e5b674f0d05e5957b4b8ed97d9c950527b5cc4ce10S3BucketC6FAEEC9" + }, + "referencetoawsstepfunctionstasksemrcontainersallservicesintegAssetParameters4129bbca38164ecb28fee8e5b674f0d05e5957b4b8ed97d9c950527b5cc4ce10S3VersionKey533AB384Ref": { + "Ref": "AssetParameters4129bbca38164ecb28fee8e5b674f0d05e5957b4b8ed97d9c950527b5cc4ce10S3VersionKeyA7EE7421" + }, + "referencetoawsstepfunctionstasksemrcontainersallservicesintegintegrationtesteksclusterDefaultVpcPrivateSubnet1SubnetFBC220C4Ref": { + "Ref": "integrationtesteksclusterDefaultVpcPrivateSubnet1Subnet4E00CAFB" + }, + "referencetoawsstepfunctionstasksemrcontainersallservicesintegintegrationtesteksclusterDefaultVpcPrivateSubnet2Subnet7E4A5E3BRef": { + "Ref": "integrationtesteksclusterDefaultVpcPrivateSubnet2Subnet0C3539A8" + }, + "referencetoawsstepfunctionstasksemrcontainersallservicesintegintegrationtesteksclusterDefaultVpcPrivateSubnet3Subnet482649D5Ref": { + "Ref": "integrationtesteksclusterDefaultVpcPrivateSubnet3SubnetD17D4AF1" + }, + "referencetoawsstepfunctionstasksemrcontainersallservicesintegintegrationtestekscluster4FFBB19EClusterSecurityGroupId": { + "Fn::GetAtt": [ + "integrationtesteksclusterE5C0ED98", + "ClusterSecurityGroupId" + ] + }, + "referencetoawsstepfunctionstasksemrcontainersallservicesintegAssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3Bucket5E7D98B4Ref": { + "Ref": "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3BucketAEADE8C7" + }, + "referencetoawsstepfunctionstasksemrcontainersallservicesintegAssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3VersionKeyC4C659E7Ref": { + "Ref": "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3VersionKeyE415415F" + }, + "referencetoawsstepfunctionstasksemrcontainersallservicesintegAssetParametersea17febe6d04c66048f3e8e060c71685c0cb53122abceff44842d27bc0d4a03eS3Bucket8CD29A22Ref": { + "Ref": "AssetParametersea17febe6d04c66048f3e8e060c71685c0cb53122abceff44842d27bc0d4a03eS3BucketD3288998" + }, + "referencetoawsstepfunctionstasksemrcontainersallservicesintegAssetParametersea17febe6d04c66048f3e8e060c71685c0cb53122abceff44842d27bc0d4a03eS3VersionKeyE130152FRef": { + "Ref": "AssetParametersea17febe6d04c66048f3e8e060c71685c0cb53122abceff44842d27bc0d4a03eS3VersionKeyB00C0565" + }, + "referencetoawsstepfunctionstasksemrcontainersallservicesintegAssetParametersdaeb79e3cee39c9b902dc0d5c780223e227ed573ea60976252947adab5fb2be1S3Bucket5978B8B5Ref": { + "Ref": "AssetParametersdaeb79e3cee39c9b902dc0d5c780223e227ed573ea60976252947adab5fb2be1S3BucketDC4B98B1" + }, + "referencetoawsstepfunctionstasksemrcontainersallservicesintegAssetParametersdaeb79e3cee39c9b902dc0d5c780223e227ed573ea60976252947adab5fb2be1S3VersionKey9D9F3D17Ref": { + "Ref": "AssetParametersdaeb79e3cee39c9b902dc0d5c780223e227ed573ea60976252947adab5fb2be1S3VersionKeyA495226F" + } + } + }, + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete" + }, + "JobExecutionRoleF19B4342": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": [ + "emr-containers.amazonaws.com", + { + "Fn::Join": [ + "", + [ + "states.", + { + "Ref": "AWS::Region" + }, + ".amazonaws.com" + ] + ] + } + ] + } + } + ], + "Version": "2012-10-17" + } + } + }, + "JobExecutionRolePolicy6968CCB9": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*", + "s3:DeleteObject*", + "s3:PutObject", + "s3:Abort*" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "StartaJobRunMonitoringBucket899C33D9", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "StartaJobRunMonitoringBucket899C33D9", + "Arn" + ] + }, + "/*" + ] + ] + } + ] + }, + { + "Action": [ + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "StartaJobRunMonitoringLogGroupD033B7AF", + "Arn" + ] + } + }, + { + "Action": "logs:DescribeLogStreams", + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "StartaJobRunMonitoringLogGroupD033B7AF", + "Arn" + ] + } + }, + { + "Action": "logs:DescribeLogGroups", + "Effect": "Allow", + "Resource": "arn:aws:logs:*:*:*" + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "JobExecutionRolePolicy6968CCB9", + "Roles": [ + { + "Fn::Select": [ + 1, + { + "Fn::Split": [ + "/", + { + "Fn::Select": [ + 5, + { + "Fn::Split": [ + ":", + { + "Fn::GetAtt": [ + "JobExecutionRoleF19B4342", + "Arn" + ] + } + ] + } + ] + } + ] + } + ] + } + ] + } + }, + "StartaJobRunMonitoringLogGroupD033B7AF": { + "Type": "AWS::Logs::LogGroup", + "Properties": { + "RetentionInDays": 731 + }, + "UpdateReplacePolicy": "Retain", + "DeletionPolicy": "Retain" + }, + "StartaJobRunMonitoringBucket899C33D9": { + "Type": "AWS::S3::Bucket", + "UpdateReplacePolicy": "Retain", + "DeletionPolicy": "Retain" + }, + "StateMachineRoleB840431D": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": { + "Fn::Join": [ + "", + [ + "states.", + { + "Ref": "AWS::Region" + }, + ".amazonaws.com" + ] + ] + } + } + } + ], + "Version": "2012-10-17" + } + } + }, + "StateMachineRoleDefaultPolicyDF1E6607": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": "emr-containers:CreateVirtualCluster", + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": "iam:CreateServiceLinkedRole", + "Condition": { + "StringLike": { + "iam:AWSServiceName": "emr-containers.amazonaws.com" + } + }, + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::", + { + "Ref": "AWS::AccountId" + }, + ":role/aws-service-role/emr-containers.amazonaws.com/AWSServiceRoleForAmazonEMRContainers" + ] + ] + } + }, + { + "Action": "emr-containers:StartJobRun", + "Condition": { + "StringEquals": { + "emr-containers:ExecutionRoleArn": { + "Fn::GetAtt": [ + "JobExecutionRoleF19B4342", + "Arn" + ] + } + } + }, + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":emr-containers:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":/virtualclusters/*" + ] + ] + } + }, + { + "Action": [ + "emr-containers:DescribeJobRun", + "emr-containers:CancelJobRun" + ], + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":emr-containers:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":/virtualclusters/*" + ] + ] + } + }, + { + "Action": "emr-containers:DeleteVirtualCluster", + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":emr-containers:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":/virtualclusters/*" + ] + ] + } + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "StateMachineRoleDefaultPolicyDF1E6607", + "Roles": [ + { + "Ref": "StateMachineRoleB840431D" + } + ] + } + }, + "StateMachine2E01A3A5": { + "Type": "AWS::StepFunctions::StateMachine", + "Properties": { + "RoleArn": { + "Fn::GetAtt": [ + "StateMachineRoleB840431D", + "Arn" + ] + }, + "DefinitionString": { + "Fn::Join": [ + "", + [ + "{\"StartAt\":\"Create a virtual Cluster\",\"States\":{\"Create a virtual Cluster\":{\"Next\":\"Start a Job Run\",\"Type\":\"Task\",\"ResultPath\":\"$.cluster\",\"Resource\":\"arn:", + { + "Ref": "AWS::Partition" + }, + ":states:::emr-containers:createVirtualCluster\",\"Parameters\":{\"Name\":\"Virtual-Cluster-Name\",\"ContainerProvider\":{\"Id\":\"", + { + "Ref": "integrationtesteksclusterE5C0ED98" + }, + "\",\"Info\":{\"EksInfo\":{\"Namespace\":\"default\"}},\"Type\":\"EKS\"}}},\"Start a Job Run\":{\"Next\":\"Delete a Virtual Cluster\",\"Type\":\"Task\",\"ResultPath\":\"$.job\",\"Resource\":\"arn:", + { + "Ref": "AWS::Partition" + }, + ":states:::emr-containers:startJobRun.sync\",\"Parameters\":{\"VirtualClusterId.$\":\"$.cluster.Id\",\"Name\":\"EMR-Containers-Job\",\"ExecutionRoleArn\":\"", + { + "Fn::GetAtt": [ + "JobExecutionRoleF19B4342", + "Arn" + ] + }, + "\",\"ReleaseLabel\":\"emr-6.2.0-latest\",\"JobDriver\":{\"SparkSubmitJobDriver\":{\"EntryPoint\":\"local:///usr/lib/spark/examples/src/main/python/pi.py\",\"EntryPointArguments\":[\"2\"],\"SparkSubmitParameters\":\"--conf spark.driver.memory=512M --conf spark.kubernetes.driver.request.cores=0.2 --conf spark.kubernetes.executor.request.cores=0.2 --conf spark.sql.shuffle.partitions=60 --conf spark.dynamicAllocation.enabled=false\"}},\"ConfigurationOverrides\":{\"ApplicationConfiguration\":[{\"Classification\":\"spark-defaults\",\"Properties\":{\"spark.executor.instances\":\"1\",\"spark.executor.memory\":\"512M\"}}],\"MonitoringConfiguration\":{\"CloudWatchMonitoringConfiguration\":{\"LogGroupName\":\"", + { + "Ref": "StartaJobRunMonitoringLogGroupD033B7AF" + }, + "\"},\"PersistentAppUI\":\"ENABLED\",\"S3MonitoringConfiguration\":{\"LogUri\":\"s3://", + { + "Ref": "StartaJobRunMonitoringBucket899C33D9" + }, + "\"}}}}},\"Delete a Virtual Cluster\":{\"End\":true,\"Type\":\"Task\",\"Resource\":\"arn:", + { + "Ref": "AWS::Partition" + }, + ":states:::emr-containers:deleteVirtualCluster\",\"Parameters\":{\"Id.$\":\"$.job.VirtualClusterId\"}}},\"TimeoutSeconds\":1200}" + ] + ] + } + }, + "DependsOn": [ + "StateMachineRoleDefaultPolicyDF1E6607", + "StateMachineRoleB840431D" + ] + } + }, + "Outputs": { + "integrationtesteksclusterConfigCommandFA814999": { + "Value": { + "Fn::Join": [ + "", + [ + "aws eks update-kubeconfig --name ", + { + "Ref": "integrationtesteksclusterE5C0ED98" + }, + " --region ", + { + "Ref": "AWS::Region" + }, + " --role-arn ", + { + "Fn::GetAtt": [ + "integrationtesteksclusterMastersRole63B9B0BF", + "Arn" + ] + } + ] + ] + } + }, + "integrationtesteksclusterGetTokenCommandD7B92682": { + "Value": { + "Fn::Join": [ + "", + [ + "aws eks get-token --cluster-name ", + { + "Ref": "integrationtesteksclusterE5C0ED98" + }, + " --region ", + { + "Ref": "AWS::Region" + }, + " --role-arn ", + { + "Fn::GetAtt": [ + "integrationtesteksclusterMastersRole63B9B0BF", + "Arn" + ] + } + ] + ] + } + }, + "stateMachineArn": { + "Value": { + "Ref": "StateMachine2E01A3A5" + } + } + }, + "Parameters": { + "AssetParameters26ac61b4195cccf80ff73f332788ad7ffaab36d81ce570340a583a8364901665S3Bucket1B280681": { + "Type": "String", + "Description": "S3 bucket for asset \"26ac61b4195cccf80ff73f332788ad7ffaab36d81ce570340a583a8364901665\"" + }, + "AssetParameters26ac61b4195cccf80ff73f332788ad7ffaab36d81ce570340a583a8364901665S3VersionKeyB1E02791": { + "Type": "String", + "Description": "S3 key for asset version \"26ac61b4195cccf80ff73f332788ad7ffaab36d81ce570340a583a8364901665\"" + }, + "AssetParameters26ac61b4195cccf80ff73f332788ad7ffaab36d81ce570340a583a8364901665ArtifactHash9EA5AC29": { + "Type": "String", + "Description": "Artifact hash for asset \"26ac61b4195cccf80ff73f332788ad7ffaab36d81ce570340a583a8364901665\"" + }, + "AssetParameters5afea6e8e6c743a8d1766f21465e28d471e56bcb95c5970054b0514bc62a3720S3Bucket3B443230": { + "Type": "String", + "Description": "S3 bucket for asset \"5afea6e8e6c743a8d1766f21465e28d471e56bcb95c5970054b0514bc62a3720\"" + }, + "AssetParameters5afea6e8e6c743a8d1766f21465e28d471e56bcb95c5970054b0514bc62a3720S3VersionKeyAA4674FB": { + "Type": "String", + "Description": "S3 key for asset version \"5afea6e8e6c743a8d1766f21465e28d471e56bcb95c5970054b0514bc62a3720\"" + }, + "AssetParameters5afea6e8e6c743a8d1766f21465e28d471e56bcb95c5970054b0514bc62a3720ArtifactHash3D7A279D": { + "Type": "String", + "Description": "Artifact hash for asset \"5afea6e8e6c743a8d1766f21465e28d471e56bcb95c5970054b0514bc62a3720\"" + }, + "AssetParametersdaeb79e3cee39c9b902dc0d5c780223e227ed573ea60976252947adab5fb2be1S3BucketDC4B98B1": { + "Type": "String", + "Description": "S3 bucket for asset \"daeb79e3cee39c9b902dc0d5c780223e227ed573ea60976252947adab5fb2be1\"" + }, + "AssetParametersdaeb79e3cee39c9b902dc0d5c780223e227ed573ea60976252947adab5fb2be1S3VersionKeyA495226F": { + "Type": "String", + "Description": "S3 key for asset version \"daeb79e3cee39c9b902dc0d5c780223e227ed573ea60976252947adab5fb2be1\"" + }, + "AssetParametersdaeb79e3cee39c9b902dc0d5c780223e227ed573ea60976252947adab5fb2be1ArtifactHashA521A16F": { + "Type": "String", + "Description": "Artifact hash for asset \"daeb79e3cee39c9b902dc0d5c780223e227ed573ea60976252947adab5fb2be1\"" + }, + "AssetParameters4129bbca38164ecb28fee8e5b674f0d05e5957b4b8ed97d9c950527b5cc4ce10S3BucketC6FAEEC9": { + "Type": "String", + "Description": "S3 bucket for asset \"4129bbca38164ecb28fee8e5b674f0d05e5957b4b8ed97d9c950527b5cc4ce10\"" + }, + "AssetParameters4129bbca38164ecb28fee8e5b674f0d05e5957b4b8ed97d9c950527b5cc4ce10S3VersionKeyA7EE7421": { + "Type": "String", + "Description": "S3 key for asset version \"4129bbca38164ecb28fee8e5b674f0d05e5957b4b8ed97d9c950527b5cc4ce10\"" + }, + "AssetParameters4129bbca38164ecb28fee8e5b674f0d05e5957b4b8ed97d9c950527b5cc4ce10ArtifactHash528547CD": { + "Type": "String", + "Description": "Artifact hash for asset \"4129bbca38164ecb28fee8e5b674f0d05e5957b4b8ed97d9c950527b5cc4ce10\"" + }, + "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3BucketAEADE8C7": { + "Type": "String", + "Description": "S3 bucket for asset \"e9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68\"" + }, + "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3VersionKeyE415415F": { + "Type": "String", + "Description": "S3 key for asset version \"e9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68\"" + }, + "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68ArtifactHashD9A515C3": { + "Type": "String", + "Description": "Artifact hash for asset \"e9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68\"" + }, + "AssetParametersea17febe6d04c66048f3e8e060c71685c0cb53122abceff44842d27bc0d4a03eS3BucketD3288998": { + "Type": "String", + "Description": "S3 bucket for asset \"ea17febe6d04c66048f3e8e060c71685c0cb53122abceff44842d27bc0d4a03e\"" + }, + "AssetParametersea17febe6d04c66048f3e8e060c71685c0cb53122abceff44842d27bc0d4a03eS3VersionKeyB00C0565": { + "Type": "String", + "Description": "S3 key for asset version \"ea17febe6d04c66048f3e8e060c71685c0cb53122abceff44842d27bc0d4a03e\"" + }, + "AssetParametersea17febe6d04c66048f3e8e060c71685c0cb53122abceff44842d27bc0d4a03eArtifactHash4654D012": { + "Type": "String", + "Description": "Artifact hash for asset \"ea17febe6d04c66048f3e8e060c71685c0cb53122abceff44842d27bc0d4a03e\"" + }, + "AssetParameters8d0ccbee43ce642958da8926fed939a9fde9d6631e9297740d640c0859ca5945S3BucketC636D050": { + "Type": "String", + "Description": "S3 bucket for asset \"8d0ccbee43ce642958da8926fed939a9fde9d6631e9297740d640c0859ca5945\"" + }, + "AssetParameters8d0ccbee43ce642958da8926fed939a9fde9d6631e9297740d640c0859ca5945S3VersionKey40C5ADFE": { + "Type": "String", + "Description": "S3 key for asset version \"8d0ccbee43ce642958da8926fed939a9fde9d6631e9297740d640c0859ca5945\"" + }, + "AssetParameters8d0ccbee43ce642958da8926fed939a9fde9d6631e9297740d640c0859ca5945ArtifactHash89120E1C": { + "Type": "String", + "Description": "Artifact hash for asset \"8d0ccbee43ce642958da8926fed939a9fde9d6631e9297740d640c0859ca5945\"" + }, + "AssetParameters249207c004483eea61658aceb796afc5f7a8b39d3b2333951c04ac8787af6d50S3BucketDBE4A868": { + "Type": "String", + "Description": "S3 bucket for asset \"249207c004483eea61658aceb796afc5f7a8b39d3b2333951c04ac8787af6d50\"" + }, + "AssetParameters249207c004483eea61658aceb796afc5f7a8b39d3b2333951c04ac8787af6d50S3VersionKeyDA4E6828": { + "Type": "String", + "Description": "S3 key for asset version \"249207c004483eea61658aceb796afc5f7a8b39d3b2333951c04ac8787af6d50\"" + }, + "AssetParameters249207c004483eea61658aceb796afc5f7a8b39d3b2333951c04ac8787af6d50ArtifactHashB10F4A4B": { + "Type": "String", + "Description": "Artifact hash for asset \"249207c004483eea61658aceb796afc5f7a8b39d3b2333951c04ac8787af6d50\"" + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-stepfunctions-tasks/test/emrcontainers/integ.job-submission-workflow.ts b/packages/@aws-cdk/aws-stepfunctions-tasks/test/emrcontainers/integ.job-submission-workflow.ts new file mode 100644 index 0000000000000..0f59e4e45e46d --- /dev/null +++ b/packages/@aws-cdk/aws-stepfunctions-tasks/test/emrcontainers/integ.job-submission-workflow.ts @@ -0,0 +1,90 @@ +import * as ec2 from '@aws-cdk/aws-ec2'; +import * as eks from '@aws-cdk/aws-eks'; +import * as iam from '@aws-cdk/aws-iam'; +import * as sfn from '@aws-cdk/aws-stepfunctions'; +import * as cdk from '@aws-cdk/core'; +import { + Classification, VirtualClusterInput, EksClusterInput, EmrContainersDeleteVirtualCluster, + EmrContainersCreateVirtualCluster, EmrContainersStartJobRun, ReleaseLabel, +} from '../../lib'; + +/** + * Stack verification steps: + * Everything in the links below must be setup for the EKS Cluster and Execution Role before running the state machine. + * @see https://docs.aws.amazon.com/emr/latest/EMR-on-EKS-DevelopmentGuide/setting-up-cluster-access.html + * @see https://docs.aws.amazon.com/emr/latest/EMR-on-EKS-DevelopmentGuide/setting-up-enable-IAM.html + * @see https://docs.aws.amazon.com/emr/latest/EMR-on-EKS-DevelopmentGuide/setting-up-trust-policy.html + * + * aws stepfunctions start-execution --state-machine-arn : should return execution arn + * aws stepfunctions describe-execution --execution-arn : should return status as SUCCEEDED + */ + +const app = new cdk.App(); +const stack = new cdk.Stack(app, 'aws-stepfunctions-tasks-emr-containers-all-services-integ'); + +const eksCluster = new eks.Cluster(stack, 'integration-test-eks-cluster', { + version: eks.KubernetesVersion.V1_21, + defaultCapacity: 3, + defaultCapacityInstance: ec2.InstanceType.of(ec2.InstanceClass.M5, ec2.InstanceSize.XLARGE), +}); + +const jobExecutionRole = new iam.Role(stack, 'JobExecutionRole', { + assumedBy: new iam.CompositePrincipal( + new iam.ServicePrincipal('emr-containers.amazonaws.com'), + new iam.ServicePrincipal('states.amazonaws.com'), + ), +}); + +const createVirtualCluster = new EmrContainersCreateVirtualCluster(stack, 'Create a virtual Cluster', { + virtualClusterName: 'Virtual-Cluster-Name', + eksCluster: EksClusterInput.fromCluster(eksCluster), + resultPath: '$.cluster', +}); + +const startJobRun = new EmrContainersStartJobRun(stack, 'Start a Job Run', { + virtualCluster: VirtualClusterInput.fromTaskInput(sfn.TaskInput.fromJsonPathAt('$.cluster.Id')), + releaseLabel: ReleaseLabel.EMR_6_2_0, + jobName: 'EMR-Containers-Job', + executionRole: iam.Role.fromRoleArn(stack, 'Job-Execution-Role', jobExecutionRole.roleArn), + jobDriver: { + sparkSubmitJobDriver: { + entryPoint: sfn.TaskInput.fromText('local:///usr/lib/spark/examples/src/main/python/pi.py'), + entryPointArguments: sfn.TaskInput.fromObject(['2']), + sparkSubmitParameters: '--conf spark.driver.memory=512M --conf spark.kubernetes.driver.request.cores=0.2 --conf spark.kubernetes.executor.request.cores=0.2 --conf spark.sql.shuffle.partitions=60 --conf spark.dynamicAllocation.enabled=false', + }, + }, + monitoring: { + logging: true, + persistentAppUI: true, + }, + applicationConfig: [{ + classification: Classification.SPARK_DEFAULTS, + properties: { + 'spark.executor.instances': '1', + 'spark.executor.memory': '512M', + }, + }], + resultPath: '$.job', +}); + + +const deleteVirtualCluster = new EmrContainersDeleteVirtualCluster(stack, 'Delete a Virtual Cluster', { + virtualClusterId: sfn.TaskInput.fromJsonPathAt('$.job.VirtualClusterId'), +}); + +const chain = sfn.Chain + .start(createVirtualCluster) + .next(startJobRun) + .next(deleteVirtualCluster); + +const sm = new sfn.StateMachine(stack, 'StateMachine', { + definition: chain, + timeout: cdk.Duration.minutes(20), +}); + +new cdk.CfnOutput(stack, 'stateMachineArn', { + value: sm.stateMachineArn, +}); + + +app.synth(); \ No newline at end of file diff --git a/packages/@aws-cdk/aws-stepfunctions-tasks/test/emrcontainers/integ.start-job-run.expected.json b/packages/@aws-cdk/aws-stepfunctions-tasks/test/emrcontainers/integ.start-job-run.expected.json new file mode 100644 index 0000000000000..3ec75b7ba0384 --- /dev/null +++ b/packages/@aws-cdk/aws-stepfunctions-tasks/test/emrcontainers/integ.start-job-run.expected.json @@ -0,0 +1,2254 @@ +{ + "Resources": { + "integrationtesteksclusterDefaultVpc395E1A86": { + "Type": "AWS::EC2::VPC", + "Properties": { + "CidrBlock": "10.0.0.0/16", + "EnableDnsHostnames": true, + "EnableDnsSupport": true, + "InstanceTenancy": "default", + "Tags": [ + { + "Key": "Name", + "Value": "aws-stepfunctions-tasks-emr-containers-start-job-run-integ-test/integration-test-eks-cluster/DefaultVpc" + } + ] + } + }, + "integrationtesteksclusterDefaultVpcPublicSubnet1Subnet58061317": { + "Type": "AWS::EC2::Subnet", + "Properties": { + "CidrBlock": "10.0.0.0/19", + "VpcId": { + "Ref": "integrationtesteksclusterDefaultVpc395E1A86" + }, + "AvailabilityZone": "test-region-1a", + "MapPublicIpOnLaunch": true, + "Tags": [ + { + "Key": "aws-cdk:subnet-name", + "Value": "Public" + }, + { + "Key": "aws-cdk:subnet-type", + "Value": "Public" + }, + { + "Key": "kubernetes.io/role/elb", + "Value": "1" + }, + { + "Key": "Name", + "Value": "aws-stepfunctions-tasks-emr-containers-start-job-run-integ-test/integration-test-eks-cluster/DefaultVpc/PublicSubnet1" + } + ] + } + }, + "integrationtesteksclusterDefaultVpcPublicSubnet1RouteTable1D5A7569": { + "Type": "AWS::EC2::RouteTable", + "Properties": { + "VpcId": { + "Ref": "integrationtesteksclusterDefaultVpc395E1A86" + }, + "Tags": [ + { + "Key": "kubernetes.io/role/elb", + "Value": "1" + }, + { + "Key": "Name", + "Value": "aws-stepfunctions-tasks-emr-containers-start-job-run-integ-test/integration-test-eks-cluster/DefaultVpc/PublicSubnet1" + } + ] + } + }, + "integrationtesteksclusterDefaultVpcPublicSubnet1RouteTableAssociation4831B6A7": { + "Type": "AWS::EC2::SubnetRouteTableAssociation", + "Properties": { + "RouteTableId": { + "Ref": "integrationtesteksclusterDefaultVpcPublicSubnet1RouteTable1D5A7569" + }, + "SubnetId": { + "Ref": "integrationtesteksclusterDefaultVpcPublicSubnet1Subnet58061317" + } + } + }, + "integrationtesteksclusterDefaultVpcPublicSubnet1DefaultRoute33CE7FC3": { + "Type": "AWS::EC2::Route", + "Properties": { + "RouteTableId": { + "Ref": "integrationtesteksclusterDefaultVpcPublicSubnet1RouteTable1D5A7569" + }, + "DestinationCidrBlock": "0.0.0.0/0", + "GatewayId": { + "Ref": "integrationtesteksclusterDefaultVpcIGW9ADAFE6F" + } + }, + "DependsOn": [ + "integrationtesteksclusterDefaultVpcVPCGWE4DC2204" + ] + }, + "integrationtesteksclusterDefaultVpcPublicSubnet1EIP62A0A17B": { + "Type": "AWS::EC2::EIP", + "Properties": { + "Domain": "vpc", + "Tags": [ + { + "Key": "kubernetes.io/role/elb", + "Value": "1" + }, + { + "Key": "Name", + "Value": "aws-stepfunctions-tasks-emr-containers-start-job-run-integ-test/integration-test-eks-cluster/DefaultVpc/PublicSubnet1" + } + ] + } + }, + "integrationtesteksclusterDefaultVpcPublicSubnet1NATGatewayC9C984F9": { + "Type": "AWS::EC2::NatGateway", + "Properties": { + "SubnetId": { + "Ref": "integrationtesteksclusterDefaultVpcPublicSubnet1Subnet58061317" + }, + "AllocationId": { + "Fn::GetAtt": [ + "integrationtesteksclusterDefaultVpcPublicSubnet1EIP62A0A17B", + "AllocationId" + ] + }, + "Tags": [ + { + "Key": "kubernetes.io/role/elb", + "Value": "1" + }, + { + "Key": "Name", + "Value": "aws-stepfunctions-tasks-emr-containers-start-job-run-integ-test/integration-test-eks-cluster/DefaultVpc/PublicSubnet1" + } + ] + } + }, + "integrationtesteksclusterDefaultVpcPublicSubnet2Subnet68EAAF11": { + "Type": "AWS::EC2::Subnet", + "Properties": { + "CidrBlock": "10.0.32.0/19", + "VpcId": { + "Ref": "integrationtesteksclusterDefaultVpc395E1A86" + }, + "AvailabilityZone": "test-region-1b", + "MapPublicIpOnLaunch": true, + "Tags": [ + { + "Key": "aws-cdk:subnet-name", + "Value": "Public" + }, + { + "Key": "aws-cdk:subnet-type", + "Value": "Public" + }, + { + "Key": "kubernetes.io/role/elb", + "Value": "1" + }, + { + "Key": "Name", + "Value": "aws-stepfunctions-tasks-emr-containers-start-job-run-integ-test/integration-test-eks-cluster/DefaultVpc/PublicSubnet2" + } + ] + } + }, + "integrationtesteksclusterDefaultVpcPublicSubnet2RouteTableA4C7B327": { + "Type": "AWS::EC2::RouteTable", + "Properties": { + "VpcId": { + "Ref": "integrationtesteksclusterDefaultVpc395E1A86" + }, + "Tags": [ + { + "Key": "kubernetes.io/role/elb", + "Value": "1" + }, + { + "Key": "Name", + "Value": "aws-stepfunctions-tasks-emr-containers-start-job-run-integ-test/integration-test-eks-cluster/DefaultVpc/PublicSubnet2" + } + ] + } + }, + "integrationtesteksclusterDefaultVpcPublicSubnet2RouteTableAssociation62710C52": { + "Type": "AWS::EC2::SubnetRouteTableAssociation", + "Properties": { + "RouteTableId": { + "Ref": "integrationtesteksclusterDefaultVpcPublicSubnet2RouteTableA4C7B327" + }, + "SubnetId": { + "Ref": "integrationtesteksclusterDefaultVpcPublicSubnet2Subnet68EAAF11" + } + } + }, + "integrationtesteksclusterDefaultVpcPublicSubnet2DefaultRoute253A231E": { + "Type": "AWS::EC2::Route", + "Properties": { + "RouteTableId": { + "Ref": "integrationtesteksclusterDefaultVpcPublicSubnet2RouteTableA4C7B327" + }, + "DestinationCidrBlock": "0.0.0.0/0", + "GatewayId": { + "Ref": "integrationtesteksclusterDefaultVpcIGW9ADAFE6F" + } + }, + "DependsOn": [ + "integrationtesteksclusterDefaultVpcVPCGWE4DC2204" + ] + }, + "integrationtesteksclusterDefaultVpcPublicSubnet2EIPFC53AC43": { + "Type": "AWS::EC2::EIP", + "Properties": { + "Domain": "vpc", + "Tags": [ + { + "Key": "kubernetes.io/role/elb", + "Value": "1" + }, + { + "Key": "Name", + "Value": "aws-stepfunctions-tasks-emr-containers-start-job-run-integ-test/integration-test-eks-cluster/DefaultVpc/PublicSubnet2" + } + ] + } + }, + "integrationtesteksclusterDefaultVpcPublicSubnet2NATGatewayE109B761": { + "Type": "AWS::EC2::NatGateway", + "Properties": { + "SubnetId": { + "Ref": "integrationtesteksclusterDefaultVpcPublicSubnet2Subnet68EAAF11" + }, + "AllocationId": { + "Fn::GetAtt": [ + "integrationtesteksclusterDefaultVpcPublicSubnet2EIPFC53AC43", + "AllocationId" + ] + }, + "Tags": [ + { + "Key": "kubernetes.io/role/elb", + "Value": "1" + }, + { + "Key": "Name", + "Value": "aws-stepfunctions-tasks-emr-containers-start-job-run-integ-test/integration-test-eks-cluster/DefaultVpc/PublicSubnet2" + } + ] + } + }, + "integrationtesteksclusterDefaultVpcPublicSubnet3Subnet4307D52D": { + "Type": "AWS::EC2::Subnet", + "Properties": { + "CidrBlock": "10.0.64.0/19", + "VpcId": { + "Ref": "integrationtesteksclusterDefaultVpc395E1A86" + }, + "AvailabilityZone": "test-region-1c", + "MapPublicIpOnLaunch": true, + "Tags": [ + { + "Key": "aws-cdk:subnet-name", + "Value": "Public" + }, + { + "Key": "aws-cdk:subnet-type", + "Value": "Public" + }, + { + "Key": "kubernetes.io/role/elb", + "Value": "1" + }, + { + "Key": "Name", + "Value": "aws-stepfunctions-tasks-emr-containers-start-job-run-integ-test/integration-test-eks-cluster/DefaultVpc/PublicSubnet3" + } + ] + } + }, + "integrationtesteksclusterDefaultVpcPublicSubnet3RouteTable51E9DA7C": { + "Type": "AWS::EC2::RouteTable", + "Properties": { + "VpcId": { + "Ref": "integrationtesteksclusterDefaultVpc395E1A86" + }, + "Tags": [ + { + "Key": "kubernetes.io/role/elb", + "Value": "1" + }, + { + "Key": "Name", + "Value": "aws-stepfunctions-tasks-emr-containers-start-job-run-integ-test/integration-test-eks-cluster/DefaultVpc/PublicSubnet3" + } + ] + } + }, + "integrationtesteksclusterDefaultVpcPublicSubnet3RouteTableAssociation14A23BAF": { + "Type": "AWS::EC2::SubnetRouteTableAssociation", + "Properties": { + "RouteTableId": { + "Ref": "integrationtesteksclusterDefaultVpcPublicSubnet3RouteTable51E9DA7C" + }, + "SubnetId": { + "Ref": "integrationtesteksclusterDefaultVpcPublicSubnet3Subnet4307D52D" + } + } + }, + "integrationtesteksclusterDefaultVpcPublicSubnet3DefaultRouteB0117918": { + "Type": "AWS::EC2::Route", + "Properties": { + "RouteTableId": { + "Ref": "integrationtesteksclusterDefaultVpcPublicSubnet3RouteTable51E9DA7C" + }, + "DestinationCidrBlock": "0.0.0.0/0", + "GatewayId": { + "Ref": "integrationtesteksclusterDefaultVpcIGW9ADAFE6F" + } + }, + "DependsOn": [ + "integrationtesteksclusterDefaultVpcVPCGWE4DC2204" + ] + }, + "integrationtesteksclusterDefaultVpcPublicSubnet3EIP33828C59": { + "Type": "AWS::EC2::EIP", + "Properties": { + "Domain": "vpc", + "Tags": [ + { + "Key": "kubernetes.io/role/elb", + "Value": "1" + }, + { + "Key": "Name", + "Value": "aws-stepfunctions-tasks-emr-containers-start-job-run-integ-test/integration-test-eks-cluster/DefaultVpc/PublicSubnet3" + } + ] + } + }, + "integrationtesteksclusterDefaultVpcPublicSubnet3NATGateway01B2F0F1": { + "Type": "AWS::EC2::NatGateway", + "Properties": { + "SubnetId": { + "Ref": "integrationtesteksclusterDefaultVpcPublicSubnet3Subnet4307D52D" + }, + "AllocationId": { + "Fn::GetAtt": [ + "integrationtesteksclusterDefaultVpcPublicSubnet3EIP33828C59", + "AllocationId" + ] + }, + "Tags": [ + { + "Key": "kubernetes.io/role/elb", + "Value": "1" + }, + { + "Key": "Name", + "Value": "aws-stepfunctions-tasks-emr-containers-start-job-run-integ-test/integration-test-eks-cluster/DefaultVpc/PublicSubnet3" + } + ] + } + }, + "integrationtesteksclusterDefaultVpcPrivateSubnet1Subnet4E00CAFB": { + "Type": "AWS::EC2::Subnet", + "Properties": { + "CidrBlock": "10.0.96.0/19", + "VpcId": { + "Ref": "integrationtesteksclusterDefaultVpc395E1A86" + }, + "AvailabilityZone": "test-region-1a", + "MapPublicIpOnLaunch": false, + "Tags": [ + { + "Key": "aws-cdk:subnet-name", + "Value": "Private" + }, + { + "Key": "aws-cdk:subnet-type", + "Value": "Private" + }, + { + "Key": "kubernetes.io/role/internal-elb", + "Value": "1" + }, + { + "Key": "Name", + "Value": "aws-stepfunctions-tasks-emr-containers-start-job-run-integ-test/integration-test-eks-cluster/DefaultVpc/PrivateSubnet1" + } + ] + } + }, + "integrationtesteksclusterDefaultVpcPrivateSubnet1RouteTable4A47F4AC": { + "Type": "AWS::EC2::RouteTable", + "Properties": { + "VpcId": { + "Ref": "integrationtesteksclusterDefaultVpc395E1A86" + }, + "Tags": [ + { + "Key": "kubernetes.io/role/internal-elb", + "Value": "1" + }, + { + "Key": "Name", + "Value": "aws-stepfunctions-tasks-emr-containers-start-job-run-integ-test/integration-test-eks-cluster/DefaultVpc/PrivateSubnet1" + } + ] + } + }, + "integrationtesteksclusterDefaultVpcPrivateSubnet1RouteTableAssociation7482DD1E": { + "Type": "AWS::EC2::SubnetRouteTableAssociation", + "Properties": { + "RouteTableId": { + "Ref": "integrationtesteksclusterDefaultVpcPrivateSubnet1RouteTable4A47F4AC" + }, + "SubnetId": { + "Ref": "integrationtesteksclusterDefaultVpcPrivateSubnet1Subnet4E00CAFB" + } + } + }, + "integrationtesteksclusterDefaultVpcPrivateSubnet1DefaultRouteCC99A72C": { + "Type": "AWS::EC2::Route", + "Properties": { + "RouteTableId": { + "Ref": "integrationtesteksclusterDefaultVpcPrivateSubnet1RouteTable4A47F4AC" + }, + "DestinationCidrBlock": "0.0.0.0/0", + "NatGatewayId": { + "Ref": "integrationtesteksclusterDefaultVpcPublicSubnet1NATGatewayC9C984F9" + } + } + }, + "integrationtesteksclusterDefaultVpcPrivateSubnet2Subnet0C3539A8": { + "Type": "AWS::EC2::Subnet", + "Properties": { + "CidrBlock": "10.0.128.0/19", + "VpcId": { + "Ref": "integrationtesteksclusterDefaultVpc395E1A86" + }, + "AvailabilityZone": "test-region-1b", + "MapPublicIpOnLaunch": false, + "Tags": [ + { + "Key": "aws-cdk:subnet-name", + "Value": "Private" + }, + { + "Key": "aws-cdk:subnet-type", + "Value": "Private" + }, + { + "Key": "kubernetes.io/role/internal-elb", + "Value": "1" + }, + { + "Key": "Name", + "Value": "aws-stepfunctions-tasks-emr-containers-start-job-run-integ-test/integration-test-eks-cluster/DefaultVpc/PrivateSubnet2" + } + ] + } + }, + "integrationtesteksclusterDefaultVpcPrivateSubnet2RouteTableD7E59903": { + "Type": "AWS::EC2::RouteTable", + "Properties": { + "VpcId": { + "Ref": "integrationtesteksclusterDefaultVpc395E1A86" + }, + "Tags": [ + { + "Key": "kubernetes.io/role/internal-elb", + "Value": "1" + }, + { + "Key": "Name", + "Value": "aws-stepfunctions-tasks-emr-containers-start-job-run-integ-test/integration-test-eks-cluster/DefaultVpc/PrivateSubnet2" + } + ] + } + }, + "integrationtesteksclusterDefaultVpcPrivateSubnet2RouteTableAssociation99F934D5": { + "Type": "AWS::EC2::SubnetRouteTableAssociation", + "Properties": { + "RouteTableId": { + "Ref": "integrationtesteksclusterDefaultVpcPrivateSubnet2RouteTableD7E59903" + }, + "SubnetId": { + "Ref": "integrationtesteksclusterDefaultVpcPrivateSubnet2Subnet0C3539A8" + } + } + }, + "integrationtesteksclusterDefaultVpcPrivateSubnet2DefaultRoute50FF167F": { + "Type": "AWS::EC2::Route", + "Properties": { + "RouteTableId": { + "Ref": "integrationtesteksclusterDefaultVpcPrivateSubnet2RouteTableD7E59903" + }, + "DestinationCidrBlock": "0.0.0.0/0", + "NatGatewayId": { + "Ref": "integrationtesteksclusterDefaultVpcPublicSubnet2NATGatewayE109B761" + } + } + }, + "integrationtesteksclusterDefaultVpcPrivateSubnet3SubnetD17D4AF1": { + "Type": "AWS::EC2::Subnet", + "Properties": { + "CidrBlock": "10.0.160.0/19", + "VpcId": { + "Ref": "integrationtesteksclusterDefaultVpc395E1A86" + }, + "AvailabilityZone": "test-region-1c", + "MapPublicIpOnLaunch": false, + "Tags": [ + { + "Key": "aws-cdk:subnet-name", + "Value": "Private" + }, + { + "Key": "aws-cdk:subnet-type", + "Value": "Private" + }, + { + "Key": "kubernetes.io/role/internal-elb", + "Value": "1" + }, + { + "Key": "Name", + "Value": "aws-stepfunctions-tasks-emr-containers-start-job-run-integ-test/integration-test-eks-cluster/DefaultVpc/PrivateSubnet3" + } + ] + } + }, + "integrationtesteksclusterDefaultVpcPrivateSubnet3RouteTableA2B42791": { + "Type": "AWS::EC2::RouteTable", + "Properties": { + "VpcId": { + "Ref": "integrationtesteksclusterDefaultVpc395E1A86" + }, + "Tags": [ + { + "Key": "kubernetes.io/role/internal-elb", + "Value": "1" + }, + { + "Key": "Name", + "Value": "aws-stepfunctions-tasks-emr-containers-start-job-run-integ-test/integration-test-eks-cluster/DefaultVpc/PrivateSubnet3" + } + ] + } + }, + "integrationtesteksclusterDefaultVpcPrivateSubnet3RouteTableAssociationB70598F3": { + "Type": "AWS::EC2::SubnetRouteTableAssociation", + "Properties": { + "RouteTableId": { + "Ref": "integrationtesteksclusterDefaultVpcPrivateSubnet3RouteTableA2B42791" + }, + "SubnetId": { + "Ref": "integrationtesteksclusterDefaultVpcPrivateSubnet3SubnetD17D4AF1" + } + } + }, + "integrationtesteksclusterDefaultVpcPrivateSubnet3DefaultRoute6DE23E60": { + "Type": "AWS::EC2::Route", + "Properties": { + "RouteTableId": { + "Ref": "integrationtesteksclusterDefaultVpcPrivateSubnet3RouteTableA2B42791" + }, + "DestinationCidrBlock": "0.0.0.0/0", + "NatGatewayId": { + "Ref": "integrationtesteksclusterDefaultVpcPublicSubnet3NATGateway01B2F0F1" + } + } + }, + "integrationtesteksclusterDefaultVpcIGW9ADAFE6F": { + "Type": "AWS::EC2::InternetGateway", + "Properties": { + "Tags": [ + { + "Key": "Name", + "Value": "aws-stepfunctions-tasks-emr-containers-start-job-run-integ-test/integration-test-eks-cluster/DefaultVpc" + } + ] + } + }, + "integrationtesteksclusterDefaultVpcVPCGWE4DC2204": { + "Type": "AWS::EC2::VPCGatewayAttachment", + "Properties": { + "VpcId": { + "Ref": "integrationtesteksclusterDefaultVpc395E1A86" + }, + "InternetGatewayId": { + "Ref": "integrationtesteksclusterDefaultVpcIGW9ADAFE6F" + } + } + }, + "integrationtesteksclusterRole03F70AF0": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "eks.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/AmazonEKSClusterPolicy" + ] + ] + } + ] + } + }, + "integrationtesteksclusterControlPlaneSecurityGroup6E92F333": { + "Type": "AWS::EC2::SecurityGroup", + "Properties": { + "GroupDescription": "EKS Control Plane Security Group", + "SecurityGroupEgress": [ + { + "CidrIp": "0.0.0.0/0", + "Description": "Allow all outbound traffic by default", + "IpProtocol": "-1" + } + ], + "VpcId": { + "Ref": "integrationtesteksclusterDefaultVpc395E1A86" + } + } + }, + "integrationtesteksclusterCreationRoleB98FE02A": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "AWS": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::", + { + "Ref": "AWS::AccountId" + }, + ":root" + ] + ] + } + } + } + ], + "Version": "2012-10-17" + } + }, + "DependsOn": [ + "integrationtesteksclusterDefaultVpcIGW9ADAFE6F", + "integrationtesteksclusterDefaultVpcPrivateSubnet1DefaultRouteCC99A72C", + "integrationtesteksclusterDefaultVpcPrivateSubnet1RouteTable4A47F4AC", + "integrationtesteksclusterDefaultVpcPrivateSubnet1RouteTableAssociation7482DD1E", + "integrationtesteksclusterDefaultVpcPrivateSubnet1Subnet4E00CAFB", + "integrationtesteksclusterDefaultVpcPrivateSubnet2DefaultRoute50FF167F", + "integrationtesteksclusterDefaultVpcPrivateSubnet2RouteTableD7E59903", + "integrationtesteksclusterDefaultVpcPrivateSubnet2RouteTableAssociation99F934D5", + "integrationtesteksclusterDefaultVpcPrivateSubnet2Subnet0C3539A8", + "integrationtesteksclusterDefaultVpcPrivateSubnet3DefaultRoute6DE23E60", + "integrationtesteksclusterDefaultVpcPrivateSubnet3RouteTableA2B42791", + "integrationtesteksclusterDefaultVpcPrivateSubnet3RouteTableAssociationB70598F3", + "integrationtesteksclusterDefaultVpcPrivateSubnet3SubnetD17D4AF1", + "integrationtesteksclusterDefaultVpcPublicSubnet1DefaultRoute33CE7FC3", + "integrationtesteksclusterDefaultVpcPublicSubnet1EIP62A0A17B", + "integrationtesteksclusterDefaultVpcPublicSubnet1NATGatewayC9C984F9", + "integrationtesteksclusterDefaultVpcPublicSubnet1RouteTable1D5A7569", + "integrationtesteksclusterDefaultVpcPublicSubnet1RouteTableAssociation4831B6A7", + "integrationtesteksclusterDefaultVpcPublicSubnet1Subnet58061317", + "integrationtesteksclusterDefaultVpcPublicSubnet2DefaultRoute253A231E", + "integrationtesteksclusterDefaultVpcPublicSubnet2EIPFC53AC43", + "integrationtesteksclusterDefaultVpcPublicSubnet2NATGatewayE109B761", + "integrationtesteksclusterDefaultVpcPublicSubnet2RouteTableA4C7B327", + "integrationtesteksclusterDefaultVpcPublicSubnet2RouteTableAssociation62710C52", + "integrationtesteksclusterDefaultVpcPublicSubnet2Subnet68EAAF11", + "integrationtesteksclusterDefaultVpcPublicSubnet3DefaultRouteB0117918", + "integrationtesteksclusterDefaultVpcPublicSubnet3EIP33828C59", + "integrationtesteksclusterDefaultVpcPublicSubnet3NATGateway01B2F0F1", + "integrationtesteksclusterDefaultVpcPublicSubnet3RouteTable51E9DA7C", + "integrationtesteksclusterDefaultVpcPublicSubnet3RouteTableAssociation14A23BAF", + "integrationtesteksclusterDefaultVpcPublicSubnet3Subnet4307D52D", + "integrationtesteksclusterDefaultVpc395E1A86", + "integrationtesteksclusterDefaultVpcVPCGWE4DC2204" + ] + }, + "integrationtesteksclusterCreationRoleDefaultPolicy5417802D": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": "iam:PassRole", + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "integrationtesteksclusterRole03F70AF0", + "Arn" + ] + } + }, + { + "Action": [ + "eks:CreateCluster", + "eks:DescribeCluster", + "eks:DescribeUpdate", + "eks:DeleteCluster", + "eks:UpdateClusterVersion", + "eks:UpdateClusterConfig", + "eks:CreateFargateProfile", + "eks:TagResource", + "eks:UntagResource" + ], + "Effect": "Allow", + "Resource": [ + "*" + ] + }, + { + "Action": [ + "eks:DescribeFargateProfile", + "eks:DeleteFargateProfile" + ], + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": [ + "iam:GetRole", + "iam:listAttachedRolePolicies" + ], + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": "iam:CreateServiceLinkedRole", + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": [ + "ec2:DescribeInstances", + "ec2:DescribeNetworkInterfaces", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:DescribeRouteTables", + "ec2:DescribeDhcpOptions", + "ec2:DescribeVpcs" + ], + "Effect": "Allow", + "Resource": "*" + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "integrationtesteksclusterCreationRoleDefaultPolicy5417802D", + "Roles": [ + { + "Ref": "integrationtesteksclusterCreationRoleB98FE02A" + } + ] + }, + "DependsOn": [ + "integrationtesteksclusterDefaultVpcIGW9ADAFE6F", + "integrationtesteksclusterDefaultVpcPrivateSubnet1DefaultRouteCC99A72C", + "integrationtesteksclusterDefaultVpcPrivateSubnet1RouteTable4A47F4AC", + "integrationtesteksclusterDefaultVpcPrivateSubnet1RouteTableAssociation7482DD1E", + "integrationtesteksclusterDefaultVpcPrivateSubnet1Subnet4E00CAFB", + "integrationtesteksclusterDefaultVpcPrivateSubnet2DefaultRoute50FF167F", + "integrationtesteksclusterDefaultVpcPrivateSubnet2RouteTableD7E59903", + "integrationtesteksclusterDefaultVpcPrivateSubnet2RouteTableAssociation99F934D5", + "integrationtesteksclusterDefaultVpcPrivateSubnet2Subnet0C3539A8", + "integrationtesteksclusterDefaultVpcPrivateSubnet3DefaultRoute6DE23E60", + "integrationtesteksclusterDefaultVpcPrivateSubnet3RouteTableA2B42791", + "integrationtesteksclusterDefaultVpcPrivateSubnet3RouteTableAssociationB70598F3", + "integrationtesteksclusterDefaultVpcPrivateSubnet3SubnetD17D4AF1", + "integrationtesteksclusterDefaultVpcPublicSubnet1DefaultRoute33CE7FC3", + "integrationtesteksclusterDefaultVpcPublicSubnet1EIP62A0A17B", + "integrationtesteksclusterDefaultVpcPublicSubnet1NATGatewayC9C984F9", + "integrationtesteksclusterDefaultVpcPublicSubnet1RouteTable1D5A7569", + "integrationtesteksclusterDefaultVpcPublicSubnet1RouteTableAssociation4831B6A7", + "integrationtesteksclusterDefaultVpcPublicSubnet1Subnet58061317", + "integrationtesteksclusterDefaultVpcPublicSubnet2DefaultRoute253A231E", + "integrationtesteksclusterDefaultVpcPublicSubnet2EIPFC53AC43", + "integrationtesteksclusterDefaultVpcPublicSubnet2NATGatewayE109B761", + "integrationtesteksclusterDefaultVpcPublicSubnet2RouteTableA4C7B327", + "integrationtesteksclusterDefaultVpcPublicSubnet2RouteTableAssociation62710C52", + "integrationtesteksclusterDefaultVpcPublicSubnet2Subnet68EAAF11", + "integrationtesteksclusterDefaultVpcPublicSubnet3DefaultRouteB0117918", + "integrationtesteksclusterDefaultVpcPublicSubnet3EIP33828C59", + "integrationtesteksclusterDefaultVpcPublicSubnet3NATGateway01B2F0F1", + "integrationtesteksclusterDefaultVpcPublicSubnet3RouteTable51E9DA7C", + "integrationtesteksclusterDefaultVpcPublicSubnet3RouteTableAssociation14A23BAF", + "integrationtesteksclusterDefaultVpcPublicSubnet3Subnet4307D52D", + "integrationtesteksclusterDefaultVpc395E1A86", + "integrationtesteksclusterDefaultVpcVPCGWE4DC2204" + ] + }, + "integrationtesteksclusterE5C0ED98": { + "Type": "Custom::AWSCDK-EKS-Cluster", + "Properties": { + "ServiceToken": { + "Fn::GetAtt": [ + "awscdkawseksClusterResourceProviderNestedStackawscdkawseksClusterResourceProviderNestedStackResource9827C454", + "Outputs.awsstepfunctionstasksemrcontainersstartjobrunintegtestawscdkawseksClusterResourceProviderframeworkonEventD439F3D7Arn" + ] + }, + "Config": { + "version": "1.21", + "roleArn": { + "Fn::GetAtt": [ + "integrationtesteksclusterRole03F70AF0", + "Arn" + ] + }, + "resourcesVpcConfig": { + "subnetIds": [ + { + "Ref": "integrationtesteksclusterDefaultVpcPublicSubnet1Subnet58061317" + }, + { + "Ref": "integrationtesteksclusterDefaultVpcPublicSubnet2Subnet68EAAF11" + }, + { + "Ref": "integrationtesteksclusterDefaultVpcPublicSubnet3Subnet4307D52D" + }, + { + "Ref": "integrationtesteksclusterDefaultVpcPrivateSubnet1Subnet4E00CAFB" + }, + { + "Ref": "integrationtesteksclusterDefaultVpcPrivateSubnet2Subnet0C3539A8" + }, + { + "Ref": "integrationtesteksclusterDefaultVpcPrivateSubnet3SubnetD17D4AF1" + } + ], + "securityGroupIds": [ + { + "Fn::GetAtt": [ + "integrationtesteksclusterControlPlaneSecurityGroup6E92F333", + "GroupId" + ] + } + ], + "endpointPublicAccess": true, + "endpointPrivateAccess": true + } + }, + "AssumeRoleArn": { + "Fn::GetAtt": [ + "integrationtesteksclusterCreationRoleB98FE02A", + "Arn" + ] + }, + "AttributesRevision": 2 + }, + "DependsOn": [ + "integrationtesteksclusterDefaultVpcIGW9ADAFE6F", + "integrationtesteksclusterDefaultVpcPrivateSubnet1DefaultRouteCC99A72C", + "integrationtesteksclusterDefaultVpcPrivateSubnet1RouteTable4A47F4AC", + "integrationtesteksclusterDefaultVpcPrivateSubnet1RouteTableAssociation7482DD1E", + "integrationtesteksclusterDefaultVpcPrivateSubnet1Subnet4E00CAFB", + "integrationtesteksclusterDefaultVpcPrivateSubnet2DefaultRoute50FF167F", + "integrationtesteksclusterDefaultVpcPrivateSubnet2RouteTableD7E59903", + "integrationtesteksclusterDefaultVpcPrivateSubnet2RouteTableAssociation99F934D5", + "integrationtesteksclusterDefaultVpcPrivateSubnet2Subnet0C3539A8", + "integrationtesteksclusterDefaultVpcPrivateSubnet3DefaultRoute6DE23E60", + "integrationtesteksclusterDefaultVpcPrivateSubnet3RouteTableA2B42791", + "integrationtesteksclusterDefaultVpcPrivateSubnet3RouteTableAssociationB70598F3", + "integrationtesteksclusterDefaultVpcPrivateSubnet3SubnetD17D4AF1", + "integrationtesteksclusterDefaultVpcPublicSubnet1DefaultRoute33CE7FC3", + "integrationtesteksclusterDefaultVpcPublicSubnet1EIP62A0A17B", + "integrationtesteksclusterDefaultVpcPublicSubnet1NATGatewayC9C984F9", + "integrationtesteksclusterDefaultVpcPublicSubnet1RouteTable1D5A7569", + "integrationtesteksclusterDefaultVpcPublicSubnet1RouteTableAssociation4831B6A7", + "integrationtesteksclusterDefaultVpcPublicSubnet1Subnet58061317", + "integrationtesteksclusterDefaultVpcPublicSubnet2DefaultRoute253A231E", + "integrationtesteksclusterDefaultVpcPublicSubnet2EIPFC53AC43", + "integrationtesteksclusterDefaultVpcPublicSubnet2NATGatewayE109B761", + "integrationtesteksclusterDefaultVpcPublicSubnet2RouteTableA4C7B327", + "integrationtesteksclusterDefaultVpcPublicSubnet2RouteTableAssociation62710C52", + "integrationtesteksclusterDefaultVpcPublicSubnet2Subnet68EAAF11", + "integrationtesteksclusterDefaultVpcPublicSubnet3DefaultRouteB0117918", + "integrationtesteksclusterDefaultVpcPublicSubnet3EIP33828C59", + "integrationtesteksclusterDefaultVpcPublicSubnet3NATGateway01B2F0F1", + "integrationtesteksclusterDefaultVpcPublicSubnet3RouteTable51E9DA7C", + "integrationtesteksclusterDefaultVpcPublicSubnet3RouteTableAssociation14A23BAF", + "integrationtesteksclusterDefaultVpcPublicSubnet3Subnet4307D52D", + "integrationtesteksclusterDefaultVpc395E1A86", + "integrationtesteksclusterDefaultVpcVPCGWE4DC2204", + "integrationtesteksclusterCreationRoleDefaultPolicy5417802D", + "integrationtesteksclusterCreationRoleB98FE02A" + ], + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete" + }, + "integrationtesteksclusterKubectlReadyBarrier0D4A21B0": { + "Type": "AWS::SSM::Parameter", + "Properties": { + "Type": "String", + "Value": "aws:cdk:eks:kubectl-ready" + }, + "DependsOn": [ + "integrationtesteksclusterCreationRoleDefaultPolicy5417802D", + "integrationtesteksclusterCreationRoleB98FE02A", + "integrationtesteksclusterE5C0ED98" + ] + }, + "integrationtesteksclusterMastersRole63B9B0BF": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "AWS": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::", + { + "Ref": "AWS::AccountId" + }, + ":root" + ] + ] + } + } + } + ], + "Version": "2012-10-17" + } + } + }, + "integrationtesteksclusterAwsAuthmanifestAEF9C6DF": { + "Type": "Custom::AWSCDK-EKS-KubernetesResource", + "Properties": { + "ServiceToken": { + "Fn::GetAtt": [ + "awscdkawseksKubectlProviderNestedStackawscdkawseksKubectlProviderNestedStackResourceA7AEBA6B", + "Outputs.awsstepfunctionstasksemrcontainersstartjobrunintegtestawscdkawseksKubectlProviderframeworkonEvent69C4EA38Arn" + ] + }, + "Manifest": { + "Fn::Join": [ + "", + [ + "[{\"apiVersion\":\"v1\",\"kind\":\"ConfigMap\",\"metadata\":{\"name\":\"aws-auth\",\"namespace\":\"kube-system\",\"labels\":{\"aws.cdk.eks/prune-c89091867a17cdada4a752b4f280c4353e38671b20\":\"\"}},\"data\":{\"mapRoles\":\"[{\\\"rolearn\\\":\\\"", + { + "Fn::GetAtt": [ + "integrationtesteksclusterMastersRole63B9B0BF", + "Arn" + ] + }, + "\\\",\\\"username\\\":\\\"", + { + "Fn::GetAtt": [ + "integrationtesteksclusterMastersRole63B9B0BF", + "Arn" + ] + }, + "\\\",\\\"groups\\\":[\\\"system:masters\\\"]},{\\\"rolearn\\\":\\\"", + { + "Fn::GetAtt": [ + "integrationtesteksclusterNodegroupDefaultCapacityNodeGroupRole75D45BA7", + "Arn" + ] + }, + "\\\",\\\"username\\\":\\\"system:node:{{EC2PrivateDNSName}}\\\",\\\"groups\\\":[\\\"system:bootstrappers\\\",\\\"system:nodes\\\"]},{\\\"rolearn\\\":\\\"arn:aws:iam::", + { + "Ref": "AWS::AccountId" + }, + ":role/AWSServiceRoleForAmazonEMRContainers\\\",\\\"username\\\":\\\"emr-containers\\\",\\\"groups\\\":[]}]\",\"mapUsers\":\"[]\",\"mapAccounts\":\"[]\"}}]" + ] + ] + }, + "ClusterName": { + "Ref": "integrationtesteksclusterE5C0ED98" + }, + "RoleArn": { + "Fn::GetAtt": [ + "integrationtesteksclusterCreationRoleB98FE02A", + "Arn" + ] + }, + "PruneLabel": "aws.cdk.eks/prune-c89091867a17cdada4a752b4f280c4353e38671b20", + "Overwrite": true + }, + "DependsOn": [ + "integrationtesteksclusterKubectlReadyBarrier0D4A21B0" + ], + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete" + }, + "integrationtesteksclusterNodegroupDefaultCapacityNodeGroupRole75D45BA7": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": { + "Fn::Join": [ + "", + [ + "ec2.", + { + "Ref": "AWS::URLSuffix" + } + ] + ] + } + } + } + ], + "Version": "2012-10-17" + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/AmazonEKSWorkerNodePolicy" + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/AmazonEKS_CNI_Policy" + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" + ] + ] + } + ] + } + }, + "integrationtesteksclusterNodegroupDefaultCapacity536CF32C": { + "Type": "AWS::EKS::Nodegroup", + "Properties": { + "ClusterName": { + "Ref": "integrationtesteksclusterE5C0ED98" + }, + "NodeRole": { + "Fn::GetAtt": [ + "integrationtesteksclusterNodegroupDefaultCapacityNodeGroupRole75D45BA7", + "Arn" + ] + }, + "Subnets": [ + { + "Ref": "integrationtesteksclusterDefaultVpcPrivateSubnet1Subnet4E00CAFB" + }, + { + "Ref": "integrationtesteksclusterDefaultVpcPrivateSubnet2Subnet0C3539A8" + }, + { + "Ref": "integrationtesteksclusterDefaultVpcPrivateSubnet3SubnetD17D4AF1" + } + ], + "AmiType": "AL2_x86_64", + "ForceUpdateEnabled": true, + "InstanceTypes": [ + "m5.xlarge" + ], + "ScalingConfig": { + "DesiredSize": 3, + "MaxSize": 3, + "MinSize": 3 + } + } + }, + "integrationtesteksclustermanifestemrRoleCCE4E328": { + "Type": "Custom::AWSCDK-EKS-KubernetesResource", + "Properties": { + "ServiceToken": { + "Fn::GetAtt": [ + "awscdkawseksKubectlProviderNestedStackawscdkawseksKubectlProviderNestedStackResourceA7AEBA6B", + "Outputs.awsstepfunctionstasksemrcontainersstartjobrunintegtestawscdkawseksKubectlProviderframeworkonEvent69C4EA38Arn" + ] + }, + "Manifest": "[{\"apiVersion\":\"rbac.authorization.k8s.io/v1\",\"kind\":\"Role\",\"metadata\":{\"name\":\"emr-containers\",\"namespace\":\"default\",\"labels\":{\"aws.cdk.eks/prune-c8cef729fffd80e01dd767818967a268148bb13a2a\":\"\"}},\"rules\":[{\"apiGroups\":[\"\"],\"resources\":[\"namespaces\"],\"verbs\":[\"get\"]},{\"apiGroups\":[\"\"],\"resources\":[\"serviceaccounts\",\"services\",\"configmaps\",\"events\",\"pods\",\"pods/log\"],\"verbs\":[\"get\",\"list\",\"watch\",\"describe\",\"create\",\"edit\",\"delete\",\"deletecollection\",\"annotate\",\"patch\",\"label\"]},{\"apiGroups\":[\"\"],\"resources\":[\"secrets\"],\"verbs\":[\"create\",\"patch\",\"delete\",\"watch\"]},{\"apiGroups\":[\"apps\"],\"resources\":[\"statefulsets\",\"deployments\"],\"verbs\":[\"get\",\"list\",\"watch\",\"describe\",\"create\",\"edit\",\"delete\",\"annotate\",\"patch\",\"label\"]},{\"apiGroups\":[\"batch\"],\"resources\":[\"jobs\"],\"verbs\":[\"get\",\"list\",\"watch\",\"describe\",\"create\",\"edit\",\"delete\",\"annotate\",\"patch\",\"label\"]},{\"apiGroups\":[\"extensions\"],\"resources\":[\"ingresses\"],\"verbs\":[\"get\",\"list\",\"watch\",\"describe\",\"create\",\"edit\",\"delete\",\"annotate\",\"patch\",\"label\"]},{\"apiGroups\":[\"rbac.authorization.k8s.io\"],\"resources\":[\"roles\",\"rolebindings\"],\"verbs\":[\"get\",\"list\",\"watch\",\"describe\",\"create\",\"edit\",\"delete\",\"deletecollection\",\"annotate\",\"patch\",\"label\"]}]}]", + "ClusterName": { + "Ref": "integrationtesteksclusterE5C0ED98" + }, + "RoleArn": { + "Fn::GetAtt": [ + "integrationtesteksclusterCreationRoleB98FE02A", + "Arn" + ] + }, + "PruneLabel": "aws.cdk.eks/prune-c8cef729fffd80e01dd767818967a268148bb13a2a" + }, + "DependsOn": [ + "integrationtesteksclusterKubectlReadyBarrier0D4A21B0" + ], + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete" + }, + "integrationtesteksclustermanifestemrRoleBind8B35D2A2": { + "Type": "Custom::AWSCDK-EKS-KubernetesResource", + "Properties": { + "ServiceToken": { + "Fn::GetAtt": [ + "awscdkawseksKubectlProviderNestedStackawscdkawseksKubectlProviderNestedStackResourceA7AEBA6B", + "Outputs.awsstepfunctionstasksemrcontainersstartjobrunintegtestawscdkawseksKubectlProviderframeworkonEvent69C4EA38Arn" + ] + }, + "Manifest": "[{\"apiVersion\":\"rbac.authorization.k8s.io/v1\",\"kind\":\"RoleBinding\",\"metadata\":{\"name\":\"emr-containers\",\"namespace\":\"default\",\"labels\":{\"aws.cdk.eks/prune-c892a3812e60d138dd377a538f9d47aace2a0a8bb6\":\"\"}},\"subjects\":[{\"kind\":\"User\",\"name\":\"emr-containers\",\"apiGroup\":\"rbac.authorization.k8s.io\"}],\"roleRef\":{\"kind\":\"Role\",\"name\":\"emr-containers\",\"apiGroup\":\"rbac.authorization.k8s.io\"}}]", + "ClusterName": { + "Ref": "integrationtesteksclusterE5C0ED98" + }, + "RoleArn": { + "Fn::GetAtt": [ + "integrationtesteksclusterCreationRoleB98FE02A", + "Arn" + ] + }, + "PruneLabel": "aws.cdk.eks/prune-c892a3812e60d138dd377a538f9d47aace2a0a8bb6" + }, + "DependsOn": [ + "integrationtesteksclusterKubectlReadyBarrier0D4A21B0", + "integrationtesteksclustermanifestemrRoleCCE4E328" + ], + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete" + }, + "awscdkawseksClusterResourceProviderNestedStackawscdkawseksClusterResourceProviderNestedStackResource9827C454": { + "Type": "AWS::CloudFormation::Stack", + "Properties": { + "TemplateURL": { + "Fn::Join": [ + "", + [ + "https://s3.", + { + "Ref": "AWS::Region" + }, + ".", + { + "Ref": "AWS::URLSuffix" + }, + "/", + { + "Ref": "AssetParameters3d62bd1f77b829ff043268ba044023fc21ba7d29ffad099d58873aa490a20591S3BucketC38C4355" + }, + "/", + { + "Fn::Select": [ + 0, + { + "Fn::Split": [ + "||", + { + "Ref": "AssetParameters3d62bd1f77b829ff043268ba044023fc21ba7d29ffad099d58873aa490a20591S3VersionKey31FFCA42" + } + ] + } + ] + }, + { + "Fn::Select": [ + 1, + { + "Fn::Split": [ + "||", + { + "Ref": "AssetParameters3d62bd1f77b829ff043268ba044023fc21ba7d29ffad099d58873aa490a20591S3VersionKey31FFCA42" + } + ] + } + ] + } + ] + ] + }, + "Parameters": { + "referencetoawsstepfunctionstasksemrcontainersstartjobrunintegtestAssetParameters26ac61b4195cccf80ff73f332788ad7ffaab36d81ce570340a583a8364901665S3Bucket47FE5F69Ref": { + "Ref": "AssetParameters26ac61b4195cccf80ff73f332788ad7ffaab36d81ce570340a583a8364901665S3Bucket1B280681" + }, + "referencetoawsstepfunctionstasksemrcontainersstartjobrunintegtestAssetParameters26ac61b4195cccf80ff73f332788ad7ffaab36d81ce570340a583a8364901665S3VersionKeyB5CD57E0Ref": { + "Ref": "AssetParameters26ac61b4195cccf80ff73f332788ad7ffaab36d81ce570340a583a8364901665S3VersionKeyB1E02791" + }, + "referencetoawsstepfunctionstasksemrcontainersstartjobrunintegtestintegrationtesteksclusterCreationRole19DB152EArn": { + "Fn::GetAtt": [ + "integrationtesteksclusterCreationRoleB98FE02A", + "Arn" + ] + }, + "referencetoawsstepfunctionstasksemrcontainersstartjobrunintegtestAssetParameters5afea6e8e6c743a8d1766f21465e28d471e56bcb95c5970054b0514bc62a3720S3Bucket7AE1E7DBRef": { + "Ref": "AssetParameters5afea6e8e6c743a8d1766f21465e28d471e56bcb95c5970054b0514bc62a3720S3Bucket3B443230" + }, + "referencetoawsstepfunctionstasksemrcontainersstartjobrunintegtestAssetParameters5afea6e8e6c743a8d1766f21465e28d471e56bcb95c5970054b0514bc62a3720S3VersionKey5842C10ARef": { + "Ref": "AssetParameters5afea6e8e6c743a8d1766f21465e28d471e56bcb95c5970054b0514bc62a3720S3VersionKeyAA4674FB" + }, + "referencetoawsstepfunctionstasksemrcontainersstartjobrunintegtestAssetParametersdaeb79e3cee39c9b902dc0d5c780223e227ed573ea60976252947adab5fb2be1S3Bucket977A6FD0Ref": { + "Ref": "AssetParametersdaeb79e3cee39c9b902dc0d5c780223e227ed573ea60976252947adab5fb2be1S3BucketDC4B98B1" + }, + "referencetoawsstepfunctionstasksemrcontainersstartjobrunintegtestAssetParametersdaeb79e3cee39c9b902dc0d5c780223e227ed573ea60976252947adab5fb2be1S3VersionKey9A35F1EFRef": { + "Ref": "AssetParametersdaeb79e3cee39c9b902dc0d5c780223e227ed573ea60976252947adab5fb2be1S3VersionKeyA495226F" + } + } + }, + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete" + }, + "awscdkawseksKubectlProviderNestedStackawscdkawseksKubectlProviderNestedStackResourceA7AEBA6B": { + "Type": "AWS::CloudFormation::Stack", + "Properties": { + "TemplateURL": { + "Fn::Join": [ + "", + [ + "https://s3.", + { + "Ref": "AWS::Region" + }, + ".", + { + "Ref": "AWS::URLSuffix" + }, + "/", + { + "Ref": "AssetParameters922360c7d159ef358ec5feeac54b70297766064bb1dc00b03a7f147d6f3a882bS3Bucket6FC76F07" + }, + "/", + { + "Fn::Select": [ + 0, + { + "Fn::Split": [ + "||", + { + "Ref": "AssetParameters922360c7d159ef358ec5feeac54b70297766064bb1dc00b03a7f147d6f3a882bS3VersionKey396AB4CB" + } + ] + } + ] + }, + { + "Fn::Select": [ + 1, + { + "Fn::Split": [ + "||", + { + "Ref": "AssetParameters922360c7d159ef358ec5feeac54b70297766064bb1dc00b03a7f147d6f3a882bS3VersionKey396AB4CB" + } + ] + } + ] + } + ] + ] + }, + "Parameters": { + "referencetoawsstepfunctionstasksemrcontainersstartjobrunintegtestintegrationtestekscluster4D8C900FArn": { + "Fn::GetAtt": [ + "integrationtesteksclusterE5C0ED98", + "Arn" + ] + }, + "referencetoawsstepfunctionstasksemrcontainersstartjobrunintegtestintegrationtesteksclusterCreationRole19DB152EArn": { + "Fn::GetAtt": [ + "integrationtesteksclusterCreationRoleB98FE02A", + "Arn" + ] + }, + "referencetoawsstepfunctionstasksemrcontainersstartjobrunintegtestAssetParameters4129bbca38164ecb28fee8e5b674f0d05e5957b4b8ed97d9c950527b5cc4ce10S3Bucket266CDCEARef": { + "Ref": "AssetParameters4129bbca38164ecb28fee8e5b674f0d05e5957b4b8ed97d9c950527b5cc4ce10S3BucketC6FAEEC9" + }, + "referencetoawsstepfunctionstasksemrcontainersstartjobrunintegtestAssetParameters4129bbca38164ecb28fee8e5b674f0d05e5957b4b8ed97d9c950527b5cc4ce10S3VersionKey06C0AAD8Ref": { + "Ref": "AssetParameters4129bbca38164ecb28fee8e5b674f0d05e5957b4b8ed97d9c950527b5cc4ce10S3VersionKeyA7EE7421" + }, + "referencetoawsstepfunctionstasksemrcontainersstartjobrunintegtestintegrationtesteksclusterDefaultVpcPrivateSubnet1SubnetDFF56EB6Ref": { + "Ref": "integrationtesteksclusterDefaultVpcPrivateSubnet1Subnet4E00CAFB" + }, + "referencetoawsstepfunctionstasksemrcontainersstartjobrunintegtestintegrationtesteksclusterDefaultVpcPrivateSubnet2Subnet0E779258Ref": { + "Ref": "integrationtesteksclusterDefaultVpcPrivateSubnet2Subnet0C3539A8" + }, + "referencetoawsstepfunctionstasksemrcontainersstartjobrunintegtestintegrationtesteksclusterDefaultVpcPrivateSubnet3SubnetA1BAE26ERef": { + "Ref": "integrationtesteksclusterDefaultVpcPrivateSubnet3SubnetD17D4AF1" + }, + "referencetoawsstepfunctionstasksemrcontainersstartjobrunintegtestintegrationtestekscluster4D8C900FClusterSecurityGroupId": { + "Fn::GetAtt": [ + "integrationtesteksclusterE5C0ED98", + "ClusterSecurityGroupId" + ] + }, + "referencetoawsstepfunctionstasksemrcontainersstartjobrunintegtestAssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3BucketFE6FEB31Ref": { + "Ref": "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3BucketAEADE8C7" + }, + "referencetoawsstepfunctionstasksemrcontainersstartjobrunintegtestAssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3VersionKeyBE97CFCARef": { + "Ref": "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3VersionKeyE415415F" + }, + "referencetoawsstepfunctionstasksemrcontainersstartjobrunintegtestAssetParametersea17febe6d04c66048f3e8e060c71685c0cb53122abceff44842d27bc0d4a03eS3BucketF38DB26BRef": { + "Ref": "AssetParametersea17febe6d04c66048f3e8e060c71685c0cb53122abceff44842d27bc0d4a03eS3BucketD3288998" + }, + "referencetoawsstepfunctionstasksemrcontainersstartjobrunintegtestAssetParametersea17febe6d04c66048f3e8e060c71685c0cb53122abceff44842d27bc0d4a03eS3VersionKey1E1E9DA8Ref": { + "Ref": "AssetParametersea17febe6d04c66048f3e8e060c71685c0cb53122abceff44842d27bc0d4a03eS3VersionKeyB00C0565" + }, + "referencetoawsstepfunctionstasksemrcontainersstartjobrunintegtestAssetParametersdaeb79e3cee39c9b902dc0d5c780223e227ed573ea60976252947adab5fb2be1S3Bucket977A6FD0Ref": { + "Ref": "AssetParametersdaeb79e3cee39c9b902dc0d5c780223e227ed573ea60976252947adab5fb2be1S3BucketDC4B98B1" + }, + "referencetoawsstepfunctionstasksemrcontainersstartjobrunintegtestAssetParametersdaeb79e3cee39c9b902dc0d5c780223e227ed573ea60976252947adab5fb2be1S3VersionKey9A35F1EFRef": { + "Ref": "AssetParametersdaeb79e3cee39c9b902dc0d5c780223e227ed573ea60976252947adab5fb2be1S3VersionKeyA495226F" + } + } + }, + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete" + }, + "VirtualCluster": { + "Type": "AWS::EMRContainers::VirtualCluster", + "Properties": { + "ContainerProvider": { + "Id": { + "Ref": "integrationtesteksclusterE5C0ED98" + }, + "Info": { + "EksInfo": { + "Namespace": "default" + } + }, + "Type": "EKS" + }, + "Name": "Virtual-Cluster-Name" + }, + "DependsOn": [ + "integrationtesteksclusterAwsAuthmanifestAEF9C6DF", + "integrationtesteksclustermanifestemrRoleBind8B35D2A2" + ] + }, + "StartaJobRunJobExecutionRole157B6BE1": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": [ + "emr-containers.amazonaws.com", + { + "Fn::Join": [ + "", + [ + "states.", + { + "Ref": "AWS::Region" + }, + ".amazonaws.com" + ] + ] + } + ] + } + } + ], + "Version": "2012-10-17" + } + } + }, + "StartaJobRunJobExecutionRoleDefaultPolicyEA7882C0": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": "logs:DescribeLogGroups", + "Effect": "Allow", + "Resource": "arn:aws:logs:*:*:*" + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "StartaJobRunJobExecutionRoleDefaultPolicyEA7882C0", + "Roles": [ + { + "Ref": "StartaJobRunJobExecutionRole157B6BE1" + } + ] + } + }, + "StartaJobRunGetEksClusterInfoCustomResourcePolicy7AA7B106": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": "emr-containers:DescribeVirtualCluster", + "Effect": "Allow", + "Resource": "*" + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "StartaJobRunGetEksClusterInfoCustomResourcePolicy7AA7B106", + "Roles": [ + { + "Ref": "AWS679f53fac002430cb0da5b7982bd2287ServiceRoleC1EA0FF2" + } + ] + } + }, + "StartaJobRunGetEksClusterInfoD0E31373": { + "Type": "Custom::AWS", + "Properties": { + "ServiceToken": { + "Fn::GetAtt": [ + "AWS679f53fac002430cb0da5b7982bd22872D164C4C", + "Arn" + ] + }, + "Create": { + "Fn::Join": [ + "", + [ + "{\"service\":\"EMRcontainers\",\"action\":\"describeVirtualCluster\",\"parameters\":{\"id\":\"", + { + "Fn::GetAtt": [ + "VirtualCluster", + "Id" + ] + }, + "\"},\"outputPaths\":[\"virtualCluster.containerProvider.info.eksInfo.namespace\",\"virtualCluster.containerProvider.id\"],\"physicalResourceId\":{\"id\":\"id\"}}" + ] + ] + }, + "InstallLatestAwsSdk": true + }, + "DependsOn": [ + "StartaJobRunGetEksClusterInfoCustomResourcePolicy7AA7B106" + ], + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete" + }, + "StartaJobRunawsclilayer110EEF0B": { + "Type": "AWS::Lambda::LayerVersion", + "Properties": { + "Content": { + "S3Bucket": { + "Ref": "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3BucketAEADE8C7" + }, + "S3Key": { + "Fn::Join": [ + "", + [ + { + "Fn::Select": [ + 0, + { + "Fn::Split": [ + "||", + { + "Ref": "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3VersionKeyE415415F" + } + ] + } + ] + }, + { + "Fn::Select": [ + 1, + { + "Fn::Split": [ + "||", + { + "Ref": "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3VersionKeyE415415F" + } + ] + } + ] + } + ] + ] + } + }, + "Description": "/opt/awscli/aws" + } + }, + "StartaJobRunCustomResourceProviderframeworkonEventServiceRole1D6E2464": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" + ] + ] + } + ] + } + }, + "StartaJobRunCustomResourceProviderframeworkonEventServiceRoleDefaultPolicy95FB1565": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": "lambda:InvokeFunction", + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "SingletonLambda8693BB64968944B69AAFB0CC9EB8757CB6182A5B", + "Arn" + ] + } + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "StartaJobRunCustomResourceProviderframeworkonEventServiceRoleDefaultPolicy95FB1565", + "Roles": [ + { + "Ref": "StartaJobRunCustomResourceProviderframeworkonEventServiceRole1D6E2464" + } + ] + } + }, + "StartaJobRunCustomResourceProviderframeworkonEventAC961165": { + "Type": "AWS::Lambda::Function", + "Properties": { + "Code": { + "S3Bucket": { + "Ref": "AssetParametersdaeb79e3cee39c9b902dc0d5c780223e227ed573ea60976252947adab5fb2be1S3BucketDC4B98B1" + }, + "S3Key": { + "Fn::Join": [ + "", + [ + { + "Fn::Select": [ + 0, + { + "Fn::Split": [ + "||", + { + "Ref": "AssetParametersdaeb79e3cee39c9b902dc0d5c780223e227ed573ea60976252947adab5fb2be1S3VersionKeyA495226F" + } + ] + } + ] + }, + { + "Fn::Select": [ + 1, + { + "Fn::Split": [ + "||", + { + "Ref": "AssetParametersdaeb79e3cee39c9b902dc0d5c780223e227ed573ea60976252947adab5fb2be1S3VersionKeyA495226F" + } + ] + } + ] + } + ] + ] + } + }, + "Role": { + "Fn::GetAtt": [ + "StartaJobRunCustomResourceProviderframeworkonEventServiceRole1D6E2464", + "Arn" + ] + }, + "Description": "AWS CDK resource provider framework - onEvent (aws-stepfunctions-tasks-emr-containers-start-job-run-integ-test/Start a Job Run/CustomResourceProvider)", + "Environment": { + "Variables": { + "USER_ON_EVENT_FUNCTION_ARN": { + "Fn::GetAtt": [ + "SingletonLambda8693BB64968944B69AAFB0CC9EB8757CB6182A5B", + "Arn" + ] + } + } + }, + "Handler": "framework.onEvent", + "Runtime": "nodejs12.x", + "Timeout": 900 + }, + "DependsOn": [ + "StartaJobRunCustomResourceProviderframeworkonEventServiceRoleDefaultPolicy95FB1565", + "StartaJobRunCustomResourceProviderframeworkonEventServiceRole1D6E2464" + ] + }, + "StartaJobRunCustomResource3BD90664": { + "Type": "AWS::CloudFormation::CustomResource", + "Properties": { + "ServiceToken": { + "Fn::GetAtt": [ + "StartaJobRunCustomResourceProviderframeworkonEventAC961165", + "Arn" + ] + }, + "eksNamespace": { + "Fn::GetAtt": [ + "StartaJobRunGetEksClusterInfoD0E31373", + "virtualCluster.containerProvider.info.eksInfo.namespace" + ] + }, + "eksClusterId": { + "Fn::GetAtt": [ + "StartaJobRunGetEksClusterInfoD0E31373", + "virtualCluster.containerProvider.id" + ] + }, + "roleName": { + "Ref": "StartaJobRunJobExecutionRole157B6BE1" + } + }, + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete" + }, + "AWS679f53fac002430cb0da5b7982bd2287ServiceRoleC1EA0FF2": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" + ] + ] + } + ] + } + }, + "AWS679f53fac002430cb0da5b7982bd22872D164C4C": { + "Type": "AWS::Lambda::Function", + "Properties": { + "Code": { + "S3Bucket": { + "Ref": "AssetParameters6ee0a36dd10d630708c265bcf7616c64030040c1bbc383b34150db74b744cad2S3BucketF482197E" + }, + "S3Key": { + "Fn::Join": [ + "", + [ + { + "Fn::Select": [ + 0, + { + "Fn::Split": [ + "||", + { + "Ref": "AssetParameters6ee0a36dd10d630708c265bcf7616c64030040c1bbc383b34150db74b744cad2S3VersionKey38B69632" + } + ] + } + ] + }, + { + "Fn::Select": [ + 1, + { + "Fn::Split": [ + "||", + { + "Ref": "AssetParameters6ee0a36dd10d630708c265bcf7616c64030040c1bbc383b34150db74b744cad2S3VersionKey38B69632" + } + ] + } + ] + } + ] + ] + } + }, + "Role": { + "Fn::GetAtt": [ + "AWS679f53fac002430cb0da5b7982bd2287ServiceRoleC1EA0FF2", + "Arn" + ] + }, + "Handler": "index.handler", + "Runtime": "nodejs12.x", + "Timeout": 120 + }, + "DependsOn": [ + "AWS679f53fac002430cb0da5b7982bd2287ServiceRoleC1EA0FF2" + ] + }, + "SingletonLambda8693BB64968944B69AAFB0CC9EB8757CServiceRoleF99BDB4C": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" + ] + ] + } + ] + } + }, + "SingletonLambda8693BB64968944B69AAFB0CC9EB8757CServiceRoleDefaultPolicy87B52EEA": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": "eks:DescribeCluster", + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":eks:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":cluster/", + { + "Fn::GetAtt": [ + "StartaJobRunGetEksClusterInfoD0E31373", + "virtualCluster.containerProvider.id" + ] + } + ] + ] + } + }, + { + "Action": [ + "iam:GetRole", + "iam:UpdateAssumeRolePolicy" + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "StartaJobRunJobExecutionRole157B6BE1", + "Arn" + ] + } + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "SingletonLambda8693BB64968944B69AAFB0CC9EB8757CServiceRoleDefaultPolicy87B52EEA", + "Roles": [ + { + "Ref": "SingletonLambda8693BB64968944B69AAFB0CC9EB8757CServiceRoleF99BDB4C" + } + ] + } + }, + "SingletonLambda8693BB64968944B69AAFB0CC9EB8757CB6182A5B": { + "Type": "AWS::Lambda::Function", + "Properties": { + "Code": { + "S3Bucket": { + "Ref": "AssetParametersb866fb0fd5a9b4215d1e23188632d74c01f3195f6f9d706134b197b400afb680S3Bucket56B5C500" + }, + "S3Key": { + "Fn::Join": [ + "", + [ + { + "Fn::Select": [ + 0, + { + "Fn::Split": [ + "||", + { + "Ref": "AssetParametersb866fb0fd5a9b4215d1e23188632d74c01f3195f6f9d706134b197b400afb680S3VersionKey966662B2" + } + ] + } + ] + }, + { + "Fn::Select": [ + 1, + { + "Fn::Split": [ + "||", + { + "Ref": "AssetParametersb866fb0fd5a9b4215d1e23188632d74c01f3195f6f9d706134b197b400afb680S3VersionKey966662B2" + } + ] + } + ] + } + ] + ] + } + }, + "Role": { + "Fn::GetAtt": [ + "SingletonLambda8693BB64968944B69AAFB0CC9EB8757CServiceRoleF99BDB4C", + "Arn" + ] + }, + "Handler": "index.handler", + "Layers": [ + { + "Ref": "StartaJobRunawsclilayer110EEF0B" + } + ], + "MemorySize": 256, + "Runtime": "python3.6", + "Timeout": 30 + }, + "DependsOn": [ + "SingletonLambda8693BB64968944B69AAFB0CC9EB8757CServiceRoleDefaultPolicy87B52EEA", + "SingletonLambda8693BB64968944B69AAFB0CC9EB8757CServiceRoleF99BDB4C" + ] + }, + "StateMachineRoleB840431D": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": { + "Fn::Join": [ + "", + [ + "states.", + { + "Ref": "AWS::Region" + }, + ".amazonaws.com" + ] + ] + } + } + } + ], + "Version": "2012-10-17" + } + } + }, + "StateMachineRoleDefaultPolicyDF1E6607": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": "emr-containers:StartJobRun", + "Condition": { + "StringEquals": { + "emr-containers:ExecutionRoleArn": { + "Fn::GetAtt": [ + "StartaJobRunJobExecutionRole157B6BE1", + "Arn" + ] + } + } + }, + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":emr-containers:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":/virtualclusters/", + { + "Fn::GetAtt": [ + "VirtualCluster", + "Id" + ] + } + ] + ] + } + }, + { + "Action": [ + "emr-containers:DescribeJobRun", + "emr-containers:CancelJobRun" + ], + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":emr-containers:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":/virtualclusters/", + { + "Fn::GetAtt": [ + "VirtualCluster", + "Id" + ] + }, + "/jobruns/*" + ] + ] + } + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "StateMachineRoleDefaultPolicyDF1E6607", + "Roles": [ + { + "Ref": "StateMachineRoleB840431D" + } + ] + } + }, + "StateMachine2E01A3A5": { + "Type": "AWS::StepFunctions::StateMachine", + "Properties": { + "RoleArn": { + "Fn::GetAtt": [ + "StateMachineRoleB840431D", + "Arn" + ] + }, + "DefinitionString": { + "Fn::Join": [ + "", + [ + "{\"StartAt\":\"Start a Job Run\",\"States\":{\"Start a Job Run\":{\"End\":true,\"Type\":\"Task\",\"Resource\":\"arn:", + { + "Ref": "AWS::Partition" + }, + ":states:::emr-containers:startJobRun.sync\",\"Parameters\":{\"VirtualClusterId\":\"", + { + "Fn::GetAtt": [ + "VirtualCluster", + "Id" + ] + }, + "\",\"Name\":\"EMR-Containers-Job\",\"ExecutionRoleArn\":\"", + { + "Fn::GetAtt": [ + "StartaJobRunJobExecutionRole157B6BE1", + "Arn" + ] + }, + "\",\"ReleaseLabel\":\"emr-6.2.0-latest\",\"JobDriver\":{\"SparkSubmitJobDriver\":{\"EntryPoint\":\"local:///usr/lib/spark/examples/src/main/python/pi.py\",\"EntryPointArguments\":[\"2\"],\"SparkSubmitParameters\":\"--conf spark.driver.memory=512M --conf spark.kubernetes.driver.request.cores=0.2 --conf spark.kubernetes.executor.request.cores=0.2 --conf spark.sql.shuffle.partitions=60 --conf spark.dynamicAllocation.enabled=false\"}},\"ConfigurationOverrides\":{\"MonitoringConfiguration\":{\"PersistentAppUI\":\"ENABLED\"}}}}},\"TimeoutSeconds\":1000}" + ] + ] + } + }, + "DependsOn": [ + "StateMachineRoleDefaultPolicyDF1E6607", + "StateMachineRoleB840431D" + ] + } + }, + "Outputs": { + "integrationtesteksclusterConfigCommandFA814999": { + "Value": { + "Fn::Join": [ + "", + [ + "aws eks update-kubeconfig --name ", + { + "Ref": "integrationtesteksclusterE5C0ED98" + }, + " --region ", + { + "Ref": "AWS::Region" + }, + " --role-arn ", + { + "Fn::GetAtt": [ + "integrationtesteksclusterMastersRole63B9B0BF", + "Arn" + ] + } + ] + ] + } + }, + "integrationtesteksclusterGetTokenCommandD7B92682": { + "Value": { + "Fn::Join": [ + "", + [ + "aws eks get-token --cluster-name ", + { + "Ref": "integrationtesteksclusterE5C0ED98" + }, + " --region ", + { + "Ref": "AWS::Region" + }, + " --role-arn ", + { + "Fn::GetAtt": [ + "integrationtesteksclusterMastersRole63B9B0BF", + "Arn" + ] + } + ] + ] + } + }, + "stateMachineArn": { + "Value": { + "Ref": "StateMachine2E01A3A5" + } + } + }, + "Parameters": { + "AssetParameters26ac61b4195cccf80ff73f332788ad7ffaab36d81ce570340a583a8364901665S3Bucket1B280681": { + "Type": "String", + "Description": "S3 bucket for asset \"26ac61b4195cccf80ff73f332788ad7ffaab36d81ce570340a583a8364901665\"" + }, + "AssetParameters26ac61b4195cccf80ff73f332788ad7ffaab36d81ce570340a583a8364901665S3VersionKeyB1E02791": { + "Type": "String", + "Description": "S3 key for asset version \"26ac61b4195cccf80ff73f332788ad7ffaab36d81ce570340a583a8364901665\"" + }, + "AssetParameters26ac61b4195cccf80ff73f332788ad7ffaab36d81ce570340a583a8364901665ArtifactHash9EA5AC29": { + "Type": "String", + "Description": "Artifact hash for asset \"26ac61b4195cccf80ff73f332788ad7ffaab36d81ce570340a583a8364901665\"" + }, + "AssetParameters5afea6e8e6c743a8d1766f21465e28d471e56bcb95c5970054b0514bc62a3720S3Bucket3B443230": { + "Type": "String", + "Description": "S3 bucket for asset \"5afea6e8e6c743a8d1766f21465e28d471e56bcb95c5970054b0514bc62a3720\"" + }, + "AssetParameters5afea6e8e6c743a8d1766f21465e28d471e56bcb95c5970054b0514bc62a3720S3VersionKeyAA4674FB": { + "Type": "String", + "Description": "S3 key for asset version \"5afea6e8e6c743a8d1766f21465e28d471e56bcb95c5970054b0514bc62a3720\"" + }, + "AssetParameters5afea6e8e6c743a8d1766f21465e28d471e56bcb95c5970054b0514bc62a3720ArtifactHash3D7A279D": { + "Type": "String", + "Description": "Artifact hash for asset \"5afea6e8e6c743a8d1766f21465e28d471e56bcb95c5970054b0514bc62a3720\"" + }, + "AssetParametersdaeb79e3cee39c9b902dc0d5c780223e227ed573ea60976252947adab5fb2be1S3BucketDC4B98B1": { + "Type": "String", + "Description": "S3 bucket for asset \"daeb79e3cee39c9b902dc0d5c780223e227ed573ea60976252947adab5fb2be1\"" + }, + "AssetParametersdaeb79e3cee39c9b902dc0d5c780223e227ed573ea60976252947adab5fb2be1S3VersionKeyA495226F": { + "Type": "String", + "Description": "S3 key for asset version \"daeb79e3cee39c9b902dc0d5c780223e227ed573ea60976252947adab5fb2be1\"" + }, + "AssetParametersdaeb79e3cee39c9b902dc0d5c780223e227ed573ea60976252947adab5fb2be1ArtifactHashA521A16F": { + "Type": "String", + "Description": "Artifact hash for asset \"daeb79e3cee39c9b902dc0d5c780223e227ed573ea60976252947adab5fb2be1\"" + }, + "AssetParameters4129bbca38164ecb28fee8e5b674f0d05e5957b4b8ed97d9c950527b5cc4ce10S3BucketC6FAEEC9": { + "Type": "String", + "Description": "S3 bucket for asset \"4129bbca38164ecb28fee8e5b674f0d05e5957b4b8ed97d9c950527b5cc4ce10\"" + }, + "AssetParameters4129bbca38164ecb28fee8e5b674f0d05e5957b4b8ed97d9c950527b5cc4ce10S3VersionKeyA7EE7421": { + "Type": "String", + "Description": "S3 key for asset version \"4129bbca38164ecb28fee8e5b674f0d05e5957b4b8ed97d9c950527b5cc4ce10\"" + }, + "AssetParameters4129bbca38164ecb28fee8e5b674f0d05e5957b4b8ed97d9c950527b5cc4ce10ArtifactHash528547CD": { + "Type": "String", + "Description": "Artifact hash for asset \"4129bbca38164ecb28fee8e5b674f0d05e5957b4b8ed97d9c950527b5cc4ce10\"" + }, + "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3BucketAEADE8C7": { + "Type": "String", + "Description": "S3 bucket for asset \"e9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68\"" + }, + "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68S3VersionKeyE415415F": { + "Type": "String", + "Description": "S3 key for asset version \"e9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68\"" + }, + "AssetParameterse9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68ArtifactHashD9A515C3": { + "Type": "String", + "Description": "Artifact hash for asset \"e9882ab123687399f934da0d45effe675ecc8ce13b40cb946f3e1d6141fe8d68\"" + }, + "AssetParametersea17febe6d04c66048f3e8e060c71685c0cb53122abceff44842d27bc0d4a03eS3BucketD3288998": { + "Type": "String", + "Description": "S3 bucket for asset \"ea17febe6d04c66048f3e8e060c71685c0cb53122abceff44842d27bc0d4a03e\"" + }, + "AssetParametersea17febe6d04c66048f3e8e060c71685c0cb53122abceff44842d27bc0d4a03eS3VersionKeyB00C0565": { + "Type": "String", + "Description": "S3 key for asset version \"ea17febe6d04c66048f3e8e060c71685c0cb53122abceff44842d27bc0d4a03e\"" + }, + "AssetParametersea17febe6d04c66048f3e8e060c71685c0cb53122abceff44842d27bc0d4a03eArtifactHash4654D012": { + "Type": "String", + "Description": "Artifact hash for asset \"ea17febe6d04c66048f3e8e060c71685c0cb53122abceff44842d27bc0d4a03e\"" + }, + "AssetParameters6ee0a36dd10d630708c265bcf7616c64030040c1bbc383b34150db74b744cad2S3BucketF482197E": { + "Type": "String", + "Description": "S3 bucket for asset \"6ee0a36dd10d630708c265bcf7616c64030040c1bbc383b34150db74b744cad2\"" + }, + "AssetParameters6ee0a36dd10d630708c265bcf7616c64030040c1bbc383b34150db74b744cad2S3VersionKey38B69632": { + "Type": "String", + "Description": "S3 key for asset version \"6ee0a36dd10d630708c265bcf7616c64030040c1bbc383b34150db74b744cad2\"" + }, + "AssetParameters6ee0a36dd10d630708c265bcf7616c64030040c1bbc383b34150db74b744cad2ArtifactHash4BE92B79": { + "Type": "String", + "Description": "Artifact hash for asset \"6ee0a36dd10d630708c265bcf7616c64030040c1bbc383b34150db74b744cad2\"" + }, + "AssetParametersb866fb0fd5a9b4215d1e23188632d74c01f3195f6f9d706134b197b400afb680S3Bucket56B5C500": { + "Type": "String", + "Description": "S3 bucket for asset \"b866fb0fd5a9b4215d1e23188632d74c01f3195f6f9d706134b197b400afb680\"" + }, + "AssetParametersb866fb0fd5a9b4215d1e23188632d74c01f3195f6f9d706134b197b400afb680S3VersionKey966662B2": { + "Type": "String", + "Description": "S3 key for asset version \"b866fb0fd5a9b4215d1e23188632d74c01f3195f6f9d706134b197b400afb680\"" + }, + "AssetParametersb866fb0fd5a9b4215d1e23188632d74c01f3195f6f9d706134b197b400afb680ArtifactHashE32BBB7E": { + "Type": "String", + "Description": "Artifact hash for asset \"b866fb0fd5a9b4215d1e23188632d74c01f3195f6f9d706134b197b400afb680\"" + }, + "AssetParameters3d62bd1f77b829ff043268ba044023fc21ba7d29ffad099d58873aa490a20591S3BucketC38C4355": { + "Type": "String", + "Description": "S3 bucket for asset \"3d62bd1f77b829ff043268ba044023fc21ba7d29ffad099d58873aa490a20591\"" + }, + "AssetParameters3d62bd1f77b829ff043268ba044023fc21ba7d29ffad099d58873aa490a20591S3VersionKey31FFCA42": { + "Type": "String", + "Description": "S3 key for asset version \"3d62bd1f77b829ff043268ba044023fc21ba7d29ffad099d58873aa490a20591\"" + }, + "AssetParameters3d62bd1f77b829ff043268ba044023fc21ba7d29ffad099d58873aa490a20591ArtifactHash2A3C4E8F": { + "Type": "String", + "Description": "Artifact hash for asset \"3d62bd1f77b829ff043268ba044023fc21ba7d29ffad099d58873aa490a20591\"" + }, + "AssetParameters922360c7d159ef358ec5feeac54b70297766064bb1dc00b03a7f147d6f3a882bS3Bucket6FC76F07": { + "Type": "String", + "Description": "S3 bucket for asset \"922360c7d159ef358ec5feeac54b70297766064bb1dc00b03a7f147d6f3a882b\"" + }, + "AssetParameters922360c7d159ef358ec5feeac54b70297766064bb1dc00b03a7f147d6f3a882bS3VersionKey396AB4CB": { + "Type": "String", + "Description": "S3 key for asset version \"922360c7d159ef358ec5feeac54b70297766064bb1dc00b03a7f147d6f3a882b\"" + }, + "AssetParameters922360c7d159ef358ec5feeac54b70297766064bb1dc00b03a7f147d6f3a882bArtifactHash2918634A": { + "Type": "String", + "Description": "Artifact hash for asset \"922360c7d159ef358ec5feeac54b70297766064bb1dc00b03a7f147d6f3a882b\"" + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-stepfunctions-tasks/test/emrcontainers/integ.start-job-run.ts b/packages/@aws-cdk/aws-stepfunctions-tasks/test/emrcontainers/integ.start-job-run.ts new file mode 100644 index 0000000000000..c7db245f5012a --- /dev/null +++ b/packages/@aws-cdk/aws-stepfunctions-tasks/test/emrcontainers/integ.start-job-run.ts @@ -0,0 +1,100 @@ +import * as ec2 from '@aws-cdk/aws-ec2'; +import * as eks from '@aws-cdk/aws-eks'; +import { AwsAuthMapping } from '@aws-cdk/aws-eks'; +import * as iam from '@aws-cdk/aws-iam'; +import * as sfn from '@aws-cdk/aws-stepfunctions'; +import * as cdk from '@aws-cdk/core'; +import { Aws } from '@aws-cdk/core'; +import { EmrContainersStartJobRun } from '../../lib'; +import { ReleaseLabel, VirtualClusterInput } from '../../lib/emrcontainers/start-job-run'; + +/** + * Stack verification steps: + * Everything in the link below must be setup before running the state machine. + * @see https://docs.aws.amazon.com/emr/latest/EMR-on-EKS-DevelopmentGuide/setting-up-enable-IAM.html + * aws stepfunctions start-execution --state-machine-arn : should return execution arn + * aws stepfunctions describe-execution --execution-arn : should return status as SUCCEEDED + */ + +const app = new cdk.App(); +const stack = new cdk.Stack(app, 'aws-stepfunctions-tasks-emr-containers-start-job-run-integ-test'); + +const eksCluster = new eks.Cluster(stack, 'integration-test-eks-cluster', { + version: eks.KubernetesVersion.V1_21, + defaultCapacity: 3, + defaultCapacityInstance: ec2.InstanceType.of(ec2.InstanceClass.M5, ec2.InstanceSize.XLARGE), +}); + +const virtualCluster = new cdk.CfnResource(stack, 'Virtual Cluster', { + type: 'AWS::EMRContainers::VirtualCluster', + properties: { + ContainerProvider: { + Id: eksCluster.clusterName, + Info: { + EksInfo: { + Namespace: 'default', + }, + }, + Type: 'EKS', + }, + Name: 'Virtual-Cluster-Name', + }, +}); + +const emrRole = eksCluster.addManifest('emrRole', { + apiVersion: 'rbac.authorization.k8s.io/v1', + kind: 'Role', + metadata: { name: 'emr-containers', namespace: 'default' }, + rules: [ + { apiGroups: [''], resources: ['namespaces'], verbs: ['get'] }, + { apiGroups: [''], resources: ['serviceaccounts', 'services', 'configmaps', 'events', 'pods', 'pods/log'], verbs: ['get', 'list', 'watch', 'describe', 'create', 'edit', 'delete', 'deletecollection', 'annotate', 'patch', 'label'] }, + { apiGroups: [''], resources: ['secrets'], verbs: ['create', 'patch', 'delete', 'watch'] }, + { apiGroups: ['apps'], resources: ['statefulsets', 'deployments'], verbs: ['get', 'list', 'watch', 'describe', 'create', 'edit', 'delete', 'annotate', 'patch', 'label'] }, + { apiGroups: ['batch'], resources: ['jobs'], verbs: ['get', 'list', 'watch', 'describe', 'create', 'edit', 'delete', 'annotate', 'patch', 'label'] }, + { apiGroups: ['extensions'], resources: ['ingresses'], verbs: ['get', 'list', 'watch', 'describe', 'create', 'edit', 'delete', 'annotate', 'patch', 'label'] }, + { apiGroups: ['rbac.authorization.k8s.io'], resources: ['roles', 'rolebindings'], verbs: ['get', 'list', 'watch', 'describe', 'create', 'edit', 'delete', 'deletecollection', 'annotate', 'patch', 'label'] }, + ], +}); + +const emrRoleBind = eksCluster.addManifest('emrRoleBind', { + apiVersion: 'rbac.authorization.k8s.io/v1', + kind: 'RoleBinding', + metadata: { name: 'emr-containers', namespace: 'default' }, + subjects: [{ kind: 'User', name: 'emr-containers', apiGroup: 'rbac.authorization.k8s.io' }], + roleRef: { kind: 'Role', name: 'emr-containers', apiGroup: 'rbac.authorization.k8s.io' }, +}); + +emrRoleBind.node.addDependency(emrRole); + +const emrServiceRole = iam.Role.fromRoleArn(stack, 'emrServiceRole', 'arn:aws:iam::'+Aws.ACCOUNT_ID+':role/AWSServiceRoleForAmazonEMRContainers'); +const authMapping: AwsAuthMapping = { groups: [], username: 'emr-containers' }; +eksCluster.awsAuth.addRoleMapping(emrServiceRole, authMapping); + +virtualCluster.node.addDependency(emrRoleBind); +virtualCluster.node.addDependency(eksCluster.awsAuth); + +const startJobRunJob = new EmrContainersStartJobRun(stack, 'Start a Job Run', { + virtualCluster: VirtualClusterInput.fromVirtualClusterId(virtualCluster.getAtt('Id').toString()), + releaseLabel: ReleaseLabel.EMR_6_2_0, + jobName: 'EMR-Containers-Job', + jobDriver: { + sparkSubmitJobDriver: { + entryPoint: sfn.TaskInput.fromText('local:///usr/lib/spark/examples/src/main/python/pi.py'), + entryPointArguments: sfn.TaskInput.fromObject(['2']), + sparkSubmitParameters: '--conf spark.driver.memory=512M --conf spark.kubernetes.driver.request.cores=0.2 --conf spark.kubernetes.executor.request.cores=0.2 --conf spark.sql.shuffle.partitions=60 --conf spark.dynamicAllocation.enabled=false', + }, + }, +}); + +const chain = sfn.Chain.start(startJobRunJob); + +const sm = new sfn.StateMachine(stack, 'StateMachine', { + definition: chain, + timeout: cdk.Duration.seconds(1000), +}); + +new cdk.CfnOutput(stack, 'stateMachineArn', { + value: sm.stateMachineArn, +}); + +app.synth(); \ No newline at end of file diff --git a/packages/@aws-cdk/aws-stepfunctions-tasks/test/emrcontainers/start-job-run.test.ts b/packages/@aws-cdk/aws-stepfunctions-tasks/test/emrcontainers/start-job-run.test.ts new file mode 100644 index 0000000000000..63dbfdc6eee48 --- /dev/null +++ b/packages/@aws-cdk/aws-stepfunctions-tasks/test/emrcontainers/start-job-run.test.ts @@ -0,0 +1,998 @@ +import { Template } from '@aws-cdk/assertions'; +import * as iam from '@aws-cdk/aws-iam'; +import * as logs from '@aws-cdk/aws-logs'; +import * as s3 from '@aws-cdk/aws-s3'; +import * as sfn from '@aws-cdk/aws-stepfunctions'; +import { Stack } from '@aws-cdk/core'; +import { EmrContainersStartJobRun, VirtualClusterInput, ReleaseLabel, ApplicationConfiguration, Classification, EmrContainersStartJobRunProps } from '../../lib/emrcontainers/start-job-run'; + +let stack: Stack; +let clusterId: string; +let defaultProps: EmrContainersStartJobRunProps; + +beforeEach(() => { + stack = new Stack(); + clusterId = 'clusterId'; + defaultProps = { + virtualCluster: VirtualClusterInput.fromTaskInput(sfn.TaskInput.fromText(clusterId)), + releaseLabel: ReleaseLabel.EMR_6_2_0, + jobDriver: { + sparkSubmitJobDriver: { + entryPoint: sfn.TaskInput.fromText('local:///usr/lib/spark/examples/src/main/python/pi.py'), + sparkSubmitParameters: '--conf spark.executor.instances=2', + }, + }, + }; +}); + +describe('Invoke EMR Containers Start Job Run with ', () => { + test('Request/Response integration pattern', () => { + // WHEN + const task = new EmrContainersStartJobRun(stack, 'EMR Containers Start Job Run', { + ...defaultProps, + integrationPattern: sfn.IntegrationPattern.REQUEST_RESPONSE, + }); + + // THEN + expect(stack.resolve(task.toStateJson())).toMatchObject({ + Type: 'Task', + Resource: { + 'Fn::Join': [ + '', + [ + 'arn:', + { + Ref: 'AWS::Partition', + }, + ':states:::emr-containers:startJobRun', + ], + ], + }, + }); + }); + + test('.sync integration pattern', () => { + // WHEN + const task = new EmrContainersStartJobRun(stack, 'EMR Containers Start Job Run', defaultProps); + // THEN + expect(stack.resolve(task.toStateJson())).toMatchObject({ + Type: 'Task', + Resource: { + 'Fn::Join': [ + '', + [ + 'arn:', + { + Ref: 'AWS::Partition', + }, + ':states:::emr-containers:startJobRun.sync', + ], + ], + }, + }); + }); + + test('virtual cluster id from payload', () => { + // WHEN + const task = new EmrContainersStartJobRun(stack, 'EMR Containers Start Job Run', { + ...defaultProps, + virtualCluster: VirtualClusterInput.fromTaskInput(sfn.TaskInput.fromJsonPathAt('$.ClusterId')), + executionRole: iam.Role.fromRoleArn(stack, 'Job Execution Role', 'arn:aws:iam::xxxxxxxxxxxx:role/JobExecutionRole'), + }); + + // THEN + expect(stack.resolve(task.toStateJson())).toMatchObject({ + Parameters: { + 'VirtualClusterId.$': '$.ClusterId', + 'ExecutionRoleArn': 'arn:aws:iam::xxxxxxxxxxxx:role/JobExecutionRole', + }, + }); + }); + + test('Tags', () => { + // WHEN + const task = new EmrContainersStartJobRun(stack, 'EMR Containers Start Job Run', { + ...defaultProps, + tags: { + key: 'value', + }, + }); + + // THEN + expect(stack.resolve(task.toStateJson())).toMatchObject({ + Parameters: { + Tags: { + key: 'value', + }, + }, + }); + }); + + + test('Application Configuration', () => { + // WHEN + const task = new EmrContainersStartJobRun(stack, 'EMR Containers Start Job Run', { + ...defaultProps, + applicationConfig: [{ + classification: Classification.SPARK_DEFAULTS, + properties: { + 'spark.executor.instances': '1', + 'spark.executor.memory': '512M', + }, + }], + }); + + // THEN + expect(stack.resolve(task.toStateJson())).toMatchObject({ + Parameters: { + ConfigurationOverrides: { + ApplicationConfiguration: [{ + Classification: Classification.SPARK_DEFAULTS.classificationStatement, + Properties: { + 'spark.executor.instances': '1', + 'spark.executor.memory': '512M', + }, + }], + }, + }, + }); + }); + + test('Application Configuration with nested app config and no properties', () => { + const properties: { [key: string]: string } = { HADOOP_DATANODE_HEAPSIZE: '2048', HADOOP_NAMENODE_OPTS: '-XX:GCTimeRatio=19' }; + const struct = { classification: new Classification('export'), properties: properties }; + const appConfigNested: ApplicationConfiguration[] = [struct]; + + const mainConfig = { classification: new Classification('hadoop-env'), nestedConfig: appConfigNested }; + const appConfig: ApplicationConfiguration[] = [mainConfig]; + // WHEN + const task = new EmrContainersStartJobRun(stack, 'EMR Containers Start Job Run', { + ...defaultProps, + applicationConfig: appConfig, + }); + + // THEN + expect(stack.resolve(task.toStateJson())).toMatchObject({ + Parameters: { + ConfigurationOverrides: { + ApplicationConfiguration: [{ + Classification: 'hadoop-env', + Configurations: [{ + Classification: 'export', + Properties: { + HADOOP_DATANODE_HEAPSIZE: '2048', + HADOOP_NAMENODE_OPTS: '-XX:GCTimeRatio=19', + }, + }], + }], + }, + }, + }); + }); + + + test('Job Execution Role', () => { + // WHEN + const task = new EmrContainersStartJobRun(stack, 'EMR Containers Start Job Run', { + ...defaultProps, + executionRole: iam.Role.fromRoleArn(stack, 'Job Execution Role', 'arn:aws:iam::xxxxxxxxxxxx:role/JobExecutionRole'), + }); + + // THEN + expect(stack.resolve(task.toStateJson())).toMatchObject({ + Parameters: { + ExecutionRoleArn: 'arn:aws:iam::xxxxxxxxxxxx:role/JobExecutionRole', + }, + }); + + // THEN + Template.fromStack(stack).resourceCountIs('AWS::CloudFormation::CustomResource', 0); + }); + + test('Virtual Cluster Input from virtualClusterId', () => { + // WHEN + const task = new EmrContainersStartJobRun(stack, 'EMR Containers Start Job Run', { + ...defaultProps, + virtualCluster: VirtualClusterInput.fromVirtualClusterId(clusterId), + }); + + // THEN + expect(stack.resolve(task.toStateJson())).toMatchObject({ + Parameters: { + VirtualClusterId: clusterId, + }, + }); + }); + + describe('Invoke EMR Containers Start Job Run with Monitoring ', () => { + test('generated automatically', () => { + // WHEN + const task = new EmrContainersStartJobRun(stack, 'EMR Containers Start Job Run', { + ...defaultProps, + monitoring: { + logging: true, + }, + }); + + // THEN + Template.fromStack(stack).hasResourceProperties('AWS::IAM::Policy', { + PolicyDocument: { + Statement: [ + { + Action: [ + 's3:GetObject*', + 's3:GetBucket*', + 's3:List*', + 's3:DeleteObject*', + 's3:PutObject*', + 's3:Abort*', + ], + Effect: 'Allow', + Resource: [ + { + 'Fn::GetAtt': [ + 'EMRContainersStartJobRunMonitoringBucket8BB3FC54', + 'Arn', + ], + }, + { + 'Fn::Join': [ + '', + [ + { + 'Fn::GetAtt': [ + 'EMRContainersStartJobRunMonitoringBucket8BB3FC54', + 'Arn', + ], + }, + '/*', + ], + ], + }, + ], + }, + { + Action: [ + 'logs:CreateLogStream', + 'logs:PutLogEvents', + ], + Effect: 'Allow', + Resource: { + 'Fn::GetAtt': [ + 'EMRContainersStartJobRunMonitoringLogGroup882D450C', + 'Arn', + ], + }, + }, + { + Action: 'logs:DescribeLogStreams', + Effect: 'Allow', + Resource: { + 'Fn::GetAtt': [ + 'EMRContainersStartJobRunMonitoringLogGroup882D450C', + 'Arn', + ], + }, + }, + { + Action: 'logs:DescribeLogGroups', + Effect: 'Allow', + Resource: 'arn:aws:logs:*:*:*', + }, + ], + Version: '2012-10-17', + }, + PolicyName: 'EMRContainersStartJobRunJobExecutionRoleDefaultPolicyCDBDF13E', + Roles: [ + { + Ref: 'EMRContainersStartJobRunJobExecutionRole40B8DD81', + }, + ], + }); + + // THEN + expect(stack.resolve(task.toStateJson())).toMatchObject({ + Parameters: { + ConfigurationOverrides: { + MonitoringConfiguration: { + CloudWatchMonitoringConfiguration: { + LogGroupName: { + Ref: 'EMRContainersStartJobRunMonitoringLogGroup882D450C', + }, + }, + PersistentAppUI: 'ENABLED', + S3MonitoringConfiguration: { + LogUri: { + 'Fn::Join': [ + '', + [ + 's3://', + { + Ref: 'EMRContainersStartJobRunMonitoringBucket8BB3FC54', + }, + ], + ], + }, + }, + }, + }, + }, + }); + }); + + test('provided from user', () => { + // WHEN + const logGroup = logs.LogGroup.fromLogGroupName(stack, 'Monitoring Group', 'providedloggroup'); + const s3Bucket = s3.Bucket.fromBucketName(stack, 'Monitoring Bucket', 'providedbucket');; + const prefixName = 'prefix'; + + const task = new EmrContainersStartJobRun(stack, 'EMR Containers Start Job Run', { + ...defaultProps, + monitoring: { + logBucket: s3Bucket, + logGroup: logGroup, + logStreamNamePrefix: prefixName, + logging: false, + }, + }); + + // THEN + Template.fromStack(stack).hasResourceProperties('AWS::IAM::Policy', { + PolicyDocument: { + Statement: [ + { + Action: [ + 's3:GetObject*', + 's3:GetBucket*', + 's3:List*', + 's3:DeleteObject*', + 's3:PutObject*', + 's3:Abort*', + ], + Effect: 'Allow', + Resource: [ + { + 'Fn::Join': [ + '', + [ + 'arn:', + { + Ref: 'AWS::Partition', + }, + ':s3:::providedbucket', + ], + ], + }, + { + 'Fn::Join': [ + '', + [ + 'arn:', + { + Ref: 'AWS::Partition', + }, + ':s3:::providedbucket/*', + ], + ], + }, + ], + }, + { + Action: [ + 'logs:CreateLogStream', + 'logs:PutLogEvents', + ], + Effect: 'Allow', + Resource: { + 'Fn::Join': [ + '', + [ + 'arn:', + { + Ref: 'AWS::Partition', + }, + ':logs:', + { + Ref: 'AWS::Region', + }, + ':', + { + Ref: 'AWS::AccountId', + }, + ':log-group:providedloggroup:*', + ], + ], + }, + }, + { + Action: 'logs:DescribeLogStreams', + Effect: 'Allow', + Resource: { + 'Fn::Join': [ + '', + [ + 'arn:', + { + Ref: 'AWS::Partition', + }, + ':logs:', + { + Ref: 'AWS::Region', + }, + ':', + { + Ref: 'AWS::AccountId', + }, + ':log-group:providedloggroup:*', + ], + ], + }, + }, + { + Action: 'logs:DescribeLogGroups', + Effect: 'Allow', + Resource: 'arn:aws:logs:*:*:*', + }, + ], + Version: '2012-10-17', + }, + PolicyName: 'EMRContainersStartJobRunJobExecutionRoleDefaultPolicyCDBDF13E', + Roles: [ + { + Ref: 'EMRContainersStartJobRunJobExecutionRole40B8DD81', + }, + ], + }); + + // THEN + expect(stack.resolve(task.toStateJson())).toMatchObject({ + Parameters: { + ConfigurationOverrides: { + MonitoringConfiguration: { + CloudWatchMonitoringConfiguration: { + LogGroupName: logGroup.logGroupName, + LogStreamNamePrefix: prefixName, + }, + S3MonitoringConfiguration: { + LogUri: 's3://' + s3Bucket.bucketName, + }, + }, + }, + ExecutionRoleArn: { + 'Fn::GetAtt': [ + 'EMRContainersStartJobRunJobExecutionRole40B8DD81', + 'Arn', + ], + }, + }, + }); + }); + + test('PersistentAppUI to be disabled when set to false', () => { + // WHEN + const task = new EmrContainersStartJobRun(stack, 'EMR Containers Start Job Run', { + ...defaultProps, + monitoring: { + persistentAppUI: false, + }, + }); + + // THEN + expect(stack.resolve(task.toStateJson())).toMatchObject({ + Parameters: { + ConfigurationOverrides: { + MonitoringConfiguration: { + PersistentAppUI: 'DISABLED', + }, + }, + }, + }); + }); + }); + + describe('Task throws if ', () => { + test('Application Configuration array is larger than 100', () => { + // WHEN + const struct = { classification: Classification.SPARK }; + const appConfig: ApplicationConfiguration[] = new Array(101).fill(struct); + + // THEN + expect(() => { + new EmrContainersStartJobRun(stack, 'Task', { + ...defaultProps, + applicationConfig: appConfig, + }); + }).toThrow(`Application configuration array must have 100 or fewer entries. Received ${appConfig.length}`); + }); + + test('Application Configuration nested configuration array is larger than 100', () => { + // WHEN + const struct = { classification: Classification.SPARK }; + let appConfig: ApplicationConfiguration[] = new Array(101).fill(struct); + + const nestedConfigStruct = { classification: Classification.SPARK, nestedConfig: appConfig }; + appConfig[0] = nestedConfigStruct; + + // THEN + expect(() => { + new EmrContainersStartJobRun(stack, 'Task', { + ...defaultProps, + applicationConfig: appConfig, + }); + }).toThrow(`Application configuration array must have 100 or fewer entries. Received ${appConfig.length}`); + }); + + test('Application Configuration properties is larger than 100 entries', () => { + // WHEN + const properties: { [key: string]: string } = {}; + for (let index = 0; index <= 100; index++) { + properties[index.toString()] = 'value'; + } + const appConfig: ApplicationConfiguration = { classification: Classification.SPARK, properties: properties }; + + expect(() => { + new EmrContainersStartJobRun(stack, 'Task', { + ...defaultProps, + applicationConfig: [appConfig], + }); + }).toThrow(`Application configuration properties must have 100 or fewer entries. Received ${Object.keys(properties).length}`); + }); + + test('Application Configuration properties is undefined and nested configuration array is undefined', () => { + // WHEN + const struct = { classification: Classification.SPARK }; + let appConfig: ApplicationConfiguration[] = new Array(1).fill(struct); + + // THEN + expect(() => { + new EmrContainersStartJobRun(stack, 'Task', { + ...defaultProps, + applicationConfig: appConfig, + }); + }).toThrow('Application configuration must have either properties or nested app configurations defined.'); + }); + + test('Entry Point is not between 1 to 256 characters in length', () => { + // WHEN + const entryPointString = 'x'.repeat(257); + + // THEN + expect(() => { + new EmrContainersStartJobRun(stack, 'Start Job Run Task', { + ...defaultProps, + jobDriver: { + sparkSubmitJobDriver: { + entryPoint: sfn.TaskInput.fromText(entryPointString), + }, + }, + }); + }).toThrow(`Entry point must be between 1 and 256 characters in length. Received ${entryPointString.length}.`); + + // THEN + expect(() => { + new EmrContainersStartJobRun(stack, 'Task', { + ...defaultProps, + jobDriver: { + sparkSubmitJobDriver: { + entryPoint: sfn.TaskInput.fromText(''), + }, + }, + }); + }).toThrow('Entry point must be between 1 and 256 characters in length. Received 0.'); + }); + + test('Entry Point Arguments is not an string array that is between 1 and 10280 entries in length', () => { + // WHEN + const entryPointArgs = sfn.TaskInput.fromObject(new Array(10281).fill('x', 10281)); + const entryPointArgsNone = sfn.TaskInput.fromObject([]); + const entryPointNumbers = sfn.TaskInput.fromObject(new Array(1).fill(1)); + const entryPointJson = sfn.TaskInput.fromText('x'); + + // THEN + expect(() => { + new EmrContainersStartJobRun(stack, 'String array error Task', { + ...defaultProps, + jobDriver: { + sparkSubmitJobDriver: { + entryPoint: sfn.TaskInput.fromText('job-location'), + entryPointArguments: entryPointNumbers, + }, + }, + }); + }).toThrow('Entry point arguments must be a string array or encoded JSON path but received object'); + + // THEN + expect(() => { + new EmrContainersStartJobRun(stack, 'JSON Path error Task', { + ...defaultProps, + jobDriver: { + sparkSubmitJobDriver: { + entryPoint: sfn.TaskInput.fromText('job-location'), + entryPointArguments: entryPointJson, + }, + }, + }); + }).toThrow('Entry point arguments must be a string array or encoded JSON path, but received a non JSON path string'); + + // THEN + expect(() => { + new EmrContainersStartJobRun(stack, 'Greater than 256 Task', { + ...defaultProps, + jobDriver: { + sparkSubmitJobDriver: { + entryPoint: sfn.TaskInput.fromText('job-location'), + entryPointArguments: entryPointArgs, + }, + }, + }); + }).toThrow(`Entry point arguments must be a string array between 1 and 10280 in length. Received ${entryPointArgs.value.length}.`); + + // THEN + expect(() => { + new EmrContainersStartJobRun(stack, 'Less than 1 Task', { + ...defaultProps, + jobDriver: { + sparkSubmitJobDriver: { + entryPoint: sfn.TaskInput.fromText('job-location'), + entryPointArguments: entryPointArgsNone, + }, + }, + }); + }).toThrow(`Entry point arguments must be a string array between 1 and 10280 in length. Received ${entryPointArgsNone.value.length}.`); + }); + + test('Spark Submit Parameters is NOT between 1 characters and 102400 characters in length', () => { + // WHEN + const sparkSubmitParam = 'x'.repeat(102401); + + // THEN + expect(() => { + new EmrContainersStartJobRun(stack, 'Spark Submit Parameter Task', { + ...defaultProps, + jobDriver: { + sparkSubmitJobDriver: { + entryPoint: sfn.TaskInput.fromText('job-location'), + sparkSubmitParameters: sparkSubmitParam, + }, + }, + }); + }).toThrow(`Spark submit parameters must be between 1 and 102400 characters in length. Received ${sparkSubmitParam.length}.`); + }); + + test('an execution role is undefined and the virtual cluster ID is not a concrete value', () => { + // WHEN + const jsonPath = '$.ClusterId'; + + // THEN + expect(() => { + new EmrContainersStartJobRun(stack, 'Task', { + ...defaultProps, + virtualCluster: VirtualClusterInput.fromTaskInput(sfn.TaskInput.fromJsonPathAt(jsonPath)), + }); + }).toThrow('Execution role cannot be undefined when the virtual cluster ID is not a concrete value. Provide an execution role with the correct trust policy'); + }); + }); + + test('Permitted role actions and resources with Start Job Run for SYNC integration pattern', () => { + // WHEN + const task = new EmrContainersStartJobRun(stack, 'EMR Containers Start Job Run', defaultProps); + + new sfn.StateMachine(stack, 'SM', { + definition: task, + }); + + // THEN + Template.fromStack(stack).hasResourceProperties('AWS::IAM::Policy', { + PolicyDocument: { + Statement: [{ + Action: 'emr-containers:StartJobRun', + Condition: { + StringEquals: { + 'emr-containers:ExecutionRoleArn': { + 'Fn::GetAtt': [ + 'EMRContainersStartJobRunJobExecutionRole40B8DD81', + 'Arn', + ], + }, + }, + }, + Effect: 'Allow', + Resource: { + 'Fn::Join': [ + '', + [ + 'arn:', + { + Ref: 'AWS::Partition', + }, + ':emr-containers:', + { + Ref: 'AWS::Region', + }, + ':', + { + Ref: 'AWS::AccountId', + }, + `:/virtualclusters/${clusterId}`, + ], + ], + }, + }, + { + Action: [ + 'emr-containers:DescribeJobRun', + 'emr-containers:CancelJobRun', + ], + Effect: 'Allow', + Resource: { + 'Fn::Join': [ + '', + [ + 'arn:', + { + Ref: 'AWS::Partition', + }, + ':emr-containers:', + { + Ref: 'AWS::Region', + }, + ':', + { + Ref: 'AWS::AccountId', + }, + `:/virtualclusters/${clusterId}/jobruns/*`, + ], + ], + }, + }], + }, + }); + }); + + test('Permitted role actions and resources with Start Job Run from payload', () => { + // WHEN + const task = new EmrContainersStartJobRun(stack, 'Task', { + ...defaultProps, + virtualCluster: VirtualClusterInput.fromTaskInput(sfn.TaskInput.fromJsonPathAt('$.ClusterId')), + executionRole: iam.Role.fromRoleArn(stack, 'Job Role', 'arn:aws:iam::xxxxxxxxxxxx:role/JobExecutionRole'), + }); + + new sfn.StateMachine(stack, 'SM', { + definition: task, + }); + + // THEN + Template.fromStack(stack).hasResourceProperties('AWS::IAM::Policy', { + PolicyDocument: { + Statement: [{ + Action: 'emr-containers:StartJobRun', + Condition: { + StringEquals: { + 'emr-containers:ExecutionRoleArn': 'arn:aws:iam::xxxxxxxxxxxx:role/JobExecutionRole', + }, + }, + Effect: 'Allow', + Resource: { + 'Fn::Join': [ + '', + [ + 'arn:', + { + Ref: 'AWS::Partition', + }, + ':emr-containers:', + { + Ref: 'AWS::Region', + }, + ':', + { + Ref: 'AWS::AccountId', + }, + ':/virtualclusters/*', + ], + ], + }, + }, + { + Action: [ + 'emr-containers:DescribeJobRun', + 'emr-containers:CancelJobRun', + ], + Effect: 'Allow', + Resource: { + 'Fn::Join': [ + '', + [ + 'arn:', + { + Ref: 'AWS::Partition', + }, + ':emr-containers:', + { + Ref: 'AWS::Region', + }, + ':', + { + Ref: 'AWS::AccountId', + }, + ':/virtualclusters/*', + ], + ], + }, + }], + }, + }); + }); + + test('Permitted role actions for Start Job Run with REQUEST/RESPONSE integration pattern', () => { + // WHEN + const task = new EmrContainersStartJobRun(stack, 'Task', { + ...defaultProps, + integrationPattern: sfn.IntegrationPattern.REQUEST_RESPONSE, + }); + + new sfn.StateMachine(stack, 'SM', { + definition: task, + }); + + // THEN + Template.fromStack(stack).hasResourceProperties('AWS::IAM::Policy', { + PolicyDocument: { + Statement: [{ + Action: 'emr-containers:StartJobRun', + Condition: { + StringEquals: { + 'emr-containers:ExecutionRoleArn': { + 'Fn::GetAtt': [ + 'TaskJobExecutionRole5D5BBA5A', + 'Arn', + ], + }, + }, + }, + Effect: 'Allow', + Resource: { + 'Fn::Join': [ + '', + [ + 'arn:', + { + Ref: 'AWS::Partition', + }, + ':emr-containers:', + { + Ref: 'AWS::Region', + }, + ':', + { + Ref: 'AWS::AccountId', + }, + `:/virtualclusters/${clusterId}`, + ], + ], + }, + }], + }, + }); + }); + + test('Custom resource is created with EMR Containers Describe Virtual Cluster invocation and has correct IAM policy permissions', () => { + // WHEN + const task = new EmrContainersStartJobRun(stack, 'Task', defaultProps); + + new sfn.StateMachine(stack, 'SM', { + definition: task, + }); + + // THEN + Template.fromStack(stack).hasResourceProperties('Custom::AWS', { + ServiceToken: { + 'Fn::GetAtt': [ + 'AWS679f53fac002430cb0da5b7982bd22872D164C4C', + 'Arn', + ], + }, + Create: '{\"service\":\"EMRcontainers\",\"action\":\"describeVirtualCluster\",\"parameters\":{\"id\":\"clusterId\"},\"outputPaths\":[\"virtualCluster.containerProvider.info.eksInfo.namespace\",\"virtualCluster.containerProvider.id\"],\"physicalResourceId\":{\"id\":\"id\"}}', + InstallLatestAwsSdk: true, + }); + + // THEN + Template.fromStack(stack).hasResourceProperties('AWS::IAM::Policy', { + PolicyDocument: { + Statement: [{ + Action: 'emr-containers:DescribeVirtualCluster', + Effect: 'Allow', + Resource: '*', + }], + }, + }); + }); + + test('Custom resource is created that has correct EKS namespace, environment, AWSCLI layer, and IAM policy permissions', () => { + // WHEN + const task = new EmrContainersStartJobRun(stack, 'Task', defaultProps); + + new sfn.StateMachine(stack, 'SM', { + definition: task, + }); + + // THEN + Template.fromStack(stack).hasResourceProperties('AWS::IAM::Policy', { + PolicyDocument: { + Statement: [ + { + Action: 'eks:DescribeCluster', + Effect: 'Allow', + Resource: { + 'Fn::Join': [ + '', + [ + 'arn:', + { + Ref: 'AWS::Partition', + }, + ':eks:', + { + Ref: 'AWS::Region', + }, + ':', + { + Ref: 'AWS::AccountId', + }, + ':cluster/', + { + 'Fn::GetAtt': [ + 'TaskGetEksClusterInfo2F156985', + 'virtualCluster.containerProvider.id', + ], + }, + ], + ], + }, + }, + { + Action: [ + 'iam:GetRole', + 'iam:UpdateAssumeRolePolicy', + ], + Effect: 'Allow', + Resource: { + 'Fn::GetAtt': [ + 'TaskJobExecutionRole5D5BBA5A', + 'Arn', + ], + }, + }, + ], + }, + }); + + // THEN + Template.fromStack(stack).hasResourceProperties('AWS::Lambda::Function', { + Handler: 'index.handler', + Layers: [ + { + Ref: 'TaskawsclilayerB1A11740', + }, + ], + MemorySize: 256, + Runtime: 'python3.6', + Timeout: 30, + }); + }); +}); + +test('Task throws if WAIT_FOR_TASK_TOKEN is supplied as service integration pattern', () => { + expect(() => { + new EmrContainersStartJobRun(stack, 'Task', { + virtualCluster: VirtualClusterInput.fromTaskInput(sfn.TaskInput.fromText(clusterId)), + releaseLabel: ReleaseLabel.EMR_6_2_0, + jobDriver: { + sparkSubmitJobDriver: { + entryPoint: sfn.TaskInput.fromText('job-location'), + }, + }, + integrationPattern: sfn.IntegrationPattern.WAIT_FOR_TASK_TOKEN, + }); + }).toThrow(/Unsupported service integration pattern. Supported Patterns: REQUEST_RESPONSE,RUN_JOB. Received: WAIT_FOR_TASK_TOKEN/); +}); \ No newline at end of file