Skip to content

Commit

Permalink
chore(migrate): enable import of resources on apps created from cdk m…
Browse files Browse the repository at this point in the history
…igrate (#28678)

Apps generated from cdk migrate with resources that aren't already part of a stack will (soon) create a migrate.json file. This file contains the list of resources that should be imported upon creation of the new app.

If this file is present and the source is either `localfile` or the ARN environment matches the deployment environment, running `cdk deploy` will:
1. Create a new stack and import the resources listed in migrate.json.
2. Apply outputs and CDKMetadata through a normal deployment.

Note: `localfile` is a placeholder value so that we can run integration tests on this change. Once some of the other in-progress work is finished, this will be updated.


----

*By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*
  • Loading branch information
TheRealAmazonKendra authored Jan 12, 2024
1 parent f1275a7 commit 811ec58
Show file tree
Hide file tree
Showing 6 changed files with 221 additions and 15 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ class YourStack extends cdk.Stack {
}
}

class ImportableStack extends cdk.Stack {
class MigrateStack extends cdk.Stack {
constructor(parent, id, props) {
super(parent, id, props);

Expand All @@ -77,11 +77,22 @@ class ImportableStack extends cdk.Stack {
new cdk.CfnOutput(this, 'QueueName', {
value: queue.queueName,
});

new cdk.CfnOutput(this, 'QueueUrl', {
value: queue.queueUrl,
});

new cdk.CfnOutput(this, 'QueueLogicalId', {
value: queue.node.defaultChild.logicalId,
});
}

}
}

class ImportableStack extends MigrateStack {
constructor(parent, id, props) {
super(parent, id, props);
new cdk.CfnWaitConditionHandle(this, 'Handle');
}
}
Expand Down Expand Up @@ -470,6 +481,8 @@ switch (stackSet) {

new ImportableStack(app, `${stackPrefix}-importable-stack`);

new MigrateStack(app, `${stackPrefix}-migrate-stack`);

new ExportValueStack(app, `${stackPrefix}-export-value-stack`);

new BundlingStage(app, `${stackPrefix}-bundling-stage`);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1213,6 +1213,45 @@ integTest('test resource import', withDefaultFixture(async (fixture) => {
}
}));

integTest('test migrate deployment for app with localfile source in migrate.json', withDefaultFixture(async (fixture) => {
const outputsFile = path.join(fixture.integTestDir, 'outputs', 'outputs.json');
await fs.mkdir(path.dirname(outputsFile), { recursive: true });

// Initial deploy
await fixture.cdkDeploy('migrate-stack', {
modEnv: { ORPHAN_TOPIC: '1' },
options: ['--outputs-file', outputsFile],
});

const outputs = JSON.parse((await fs.readFile(outputsFile, { encoding: 'utf-8' })).toString());
const stackName = fixture.fullStackName('migrate-stack');
const queueName = outputs[stackName].QueueName;
const queueUrl = outputs[stackName].QueueUrl;
const queueLogicalId = outputs[stackName].QueueLogicalId;
fixture.log(`Created queue ${queueUrl} in stack ${fixture.fullStackName}`);

// Write the migrate file based on the ID from step one, then deploy the app with migrate
const migrateFile = path.join(fixture.integTestDir, 'migrate.json');
await fs.writeFile(
migrateFile, JSON.stringify(
{ Source: 'localfile', Resources: [{ ResourceType: 'AWS::SQS::Queue', LogicalResourceId: queueLogicalId, ResourceIdentifier: { QueueUrl: queueUrl } }] },
),
{ encoding: 'utf-8' },
);

await fixture.cdkDestroy('migrate-stack');
fixture.log(`Deleted stack ${fixture.fullStackName}, orphaning ${queueName}`);

// Create new stack from existing queue
try {
fixture.log(`Deploying new stack ${fixture.fullStackName}, migrating ${queueName} into stack`);
await fixture.cdkDeploy('migrate-stack');
} finally {
// Cleanup
await fixture.cdkDestroy('migrate-stack');
}
}));

integTest('hotswap deployment supports Lambda function\'s description and environment variables', withDefaultFixture(async (fixture) => {
// GIVEN
const stackArn = await fixture.cdkDeploy('lambda-hotswap', {
Expand Down
9 changes: 8 additions & 1 deletion packages/aws-cdk/lib/api/deployments.ts
Original file line number Diff line number Diff line change
Expand Up @@ -317,6 +317,13 @@ export class Deployments {
this.environmentResources = new EnvironmentResourcesRegistry(props.toolkitStackName);
}

/**
* Resolves the environment for a stack.
*/
public async resolveEnvironment(stack: cxapi.CloudFormationStackArtifact): Promise<cxapi.Environment> {
return this.sdkProvider.resolveEnvironment(stack.environment);
}

public async readCurrentTemplateWithNestedStacks(
rootStackArtifact: cxapi.CloudFormationStackArtifact,
retrieveProcessedTemplate: boolean = false,
Expand Down Expand Up @@ -470,7 +477,7 @@ export class Deployments {
throw new Error(`The stack ${stack.displayName} does not have an environment`);
}

const resolvedEnvironment = await this.sdkProvider.resolveEnvironment(stack.environment);
const resolvedEnvironment = await this.resolveEnvironment(stack);

// Substitute any placeholders with information about the current environment
const arns = await replaceEnvPlaceholders({
Expand Down
65 changes: 61 additions & 4 deletions packages/aws-cdk/lib/cdk-toolkit.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import { Deployments } from './api/deployments';
import { HotswapMode } from './api/hotswap/common';
import { findCloudWatchLogGroups } from './api/logs/find-cloudwatch-logs';
import { CloudWatchLogEventMonitor } from './api/logs/logs-monitor';
import { createDiffChangeSet } from './api/util/cloudformation';
import { createDiffChangeSet, ResourcesToImport } from './api/util/cloudformation';
import { StackActivityProgress } from './api/util/cloudformation/stack-activity-monitor';
import { generateCdkApp, generateStack, readFromPath, readFromStack, setEnvironment, validateSourceOptions } from './commands/migrate';
import { printSecurityDiff, printStackDiff, RequireApproval } from './diff';
Expand Down Expand Up @@ -205,6 +205,8 @@ export class CdkToolkit {
const elapsedSynthTime = new Date().getTime() - startSynthTime;
print('\n✨ Synthesis time: %ss\n', formatTime(elapsedSynthTime));

await this.tryMigrateResources(stackCollection, options);

const requireApproval = options.requireApproval ?? RequireApproval.Broadening;

const parameterMap = buildParameterMap(options.parameters);
Expand Down Expand Up @@ -539,9 +541,7 @@ export class CdkToolkit {
// Import the resources according to the given mapping
print('%s: importing resources into stack...', chalk.bold(stack.displayName));
const tags = tagsForStack(stack);
await resourceImporter.importResources(actualImport, {
stack,
deployName: stack.stackName,
await resourceImporter.importResourcesFromMap(actualImport, {
roleArn: options.roleArn,
toolkitStackName: options.toolkitStackName,
tags,
Expand Down Expand Up @@ -874,6 +874,63 @@ export class CdkToolkit {
stackName: assetNode.parentStack.stackName,
}));
}

/**
* Checks to see if a migrate.json file exists. If it does and the source is either `filepath` or
* is in the same environment as the stack deployment, a new stack is created and the resources are
* migrated to the stack using an IMPORT changeset. The normal deployment will resume after this is complete
* to add back in any outputs and the CDKMetadata.
*/
private async tryMigrateResources(stacks: StackCollection, options: DeployOptions): Promise<void> {
const stack = stacks.stackArtifacts[0];
const migrateDeployment = new ResourceImporter(stack, this.props.deployments);
const resourcesToImport = await this.tryGetResources(migrateDeployment);

if (resourcesToImport) {
print('%s: creating stack for resource migration...', chalk.bold(stack.displayName));
print('%s: importing resources into stack...', chalk.bold(stack.displayName));

await this.performResourceMigration(migrateDeployment, resourcesToImport, options);

fs.rmSync('migrate.json');
print('%s: applying CDKMetadata and Outputs to stack (if applicable)...', chalk.bold(stack.displayName));
}
}

/**
* Creates a new stack with just the resources to be migrated
*/
private async performResourceMigration(migrateDeployment: ResourceImporter, resourcesToImport: ResourcesToImport, options: DeployOptions) {
const startDeployTime = new Date().getTime();
let elapsedDeployTime = 0;

// Initial Deployment
await migrateDeployment.importResourcesFromMigrate(resourcesToImport, {
roleArn: options.roleArn,
toolkitStackName: options.toolkitStackName,
deploymentMethod: options.deploymentMethod,
usePreviousParameters: true,
progress: options.progress,
rollback: options.rollback,
});

elapsedDeployTime = new Date().getTime() - startDeployTime;
print('\n✨ Resource migration time: %ss\n', formatTime(elapsedDeployTime));
}

private async tryGetResources(migrateDeployment: ResourceImporter) {
try {
const migrateFile = fs.readJsonSync('migrate.json', { encoding: 'utf-8' });
const sourceEnv = (migrateFile.Source as string).split(':');
const environment = await migrateDeployment.resolveEnvironment();
if (sourceEnv[0] === 'localfile' ||
(sourceEnv[4] === environment.account && sourceEnv[3] === environment.region)) {
return migrateFile.Resources;
}
} catch (e) {
// Nothing to do
}
}
}

export interface DiffOptions {
Expand Down
58 changes: 53 additions & 5 deletions packages/aws-cdk/lib/import.ts
Original file line number Diff line number Diff line change
@@ -1,13 +1,23 @@
import { DeployOptions } from '@aws-cdk/cloud-assembly-schema';
import * as cfnDiff from '@aws-cdk/cloudformation-diff';
import { ResourceDifference } from '@aws-cdk/cloudformation-diff';
import * as cxapi from '@aws-cdk/cx-api';
import * as chalk from 'chalk';
import * as fs from 'fs-extra';
import * as promptly from 'promptly';
import { Deployments, DeployStackOptions } from './api/deployments';
import { DeploymentMethod } from './api';
import { Deployments } from './api/deployments';
import { ResourceIdentifierProperties, ResourcesToImport } from './api/util/cloudformation';
import { StackActivityProgress } from './api/util/cloudformation/stack-activity-monitor';
import { Tag } from './cdk-toolkit';
import { error, print, success, warning } from './logging';

export interface ImportDeploymentOptions extends DeployOptions {
deploymentMethod?: DeploymentMethod;
progress?: StackActivityProgress;
tags?: Tag[];
}

/**
* Set of parameters that uniquely identify a physical resource of a given type
* for the import operation, example:
Expand Down Expand Up @@ -112,24 +122,44 @@ export class ResourceImporter {
* @param importMap Mapping from CDK construct tree path to physical resource import identifiers
* @param options Options to pass to CloudFormation deploy operation
*/
public async importResources(importMap: ImportMap, options: DeployStackOptions) {
public async importResourcesFromMap(importMap: ImportMap, options: ImportDeploymentOptions) {
const resourcesToImport: ResourcesToImport = await this.makeResourcesToImport(importMap);
const updatedTemplate = await this.currentTemplateWithAdditions(importMap.importResources);

await this.importResources(updatedTemplate, resourcesToImport, options);
}

/**
* Based on the app and resources file generated by cdk migrate. Removes all items from the template that
* cannot be included in an import change-set for new stacks and performs the import operation,
* creating the new stack.
*
* @param resourcesToImport The mapping created by cdk migrate
* @param options Options to pass to CloudFormation deploy operation
*/
public async importResourcesFromMigrate(resourcesToImport: ResourcesToImport, options: ImportDeploymentOptions) {
const updatedTemplate = this.removeNonImportResources();

await this.importResources(updatedTemplate, resourcesToImport, options);
}

private async importResources(overrideTemplate: any, resourcesToImport: ResourcesToImport, options: ImportDeploymentOptions) {
try {
const result = await this.cfn.deployStack({
stack: this.stack,
deployName: this.stack.stackName,
...options,
overrideTemplate: updatedTemplate,
overrideTemplate,
resourcesToImport,
});

const message = result.noOp
? ' ✅ %s (no changes)'
: ' ✅ %s';

success('\n' + message, options.stack.displayName);
success('\n' + message, this.stack.displayName);
} catch (e) {
error('\n ❌ %s failed: %s', chalk.bold(options.stack.displayName), e);
error('\n ❌ %s failed: %s', chalk.bold(this.stack.displayName), e);
throw e;
}
}
Expand Down Expand Up @@ -176,6 +206,13 @@ export class ResourceImporter {
};
}

/**
* Resolves the environment of a stack.
*/
public async resolveEnvironment(): Promise<cxapi.Environment> {
return this.cfn.resolveEnvironment(this.stack);
}

/**
* Get currently deployed template of the given stack (SINGLETON)
*
Expand Down Expand Up @@ -342,6 +379,17 @@ export class ResourceImporter {
private describeResource(logicalId: string): string {
return this.stack.template?.Resources?.[logicalId]?.Metadata?.['aws:cdk:path'] ?? logicalId;
}

/**
* Removes CDKMetadata and Outputs in the template so that only resources for importing are left.
* @returns template with import resources only
*/
private removeNonImportResources() {
const template = this.stack.template;
delete template.Resources.CDKMetadata;
delete template.Outputs;
return template;
}
}

/**
Expand Down
50 changes: 46 additions & 4 deletions packages/aws-cdk/test/import.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -164,9 +164,7 @@ test('asks human to confirm automic import if identifier is in template', async
};

// WHEN
await importer.importResources(importMap, {
stack: STACK_WITH_QUEUE,
});
await importer.importResourcesFromMap(importMap, {});

expect(createChangeSetInput?.ResourcesToImport).toEqual([
{
Expand All @@ -177,6 +175,50 @@ test('asks human to confirm automic import if identifier is in template', async
]);
});

test('importing resources from migrate strips cdk metadata and outputs', async () => {
// GIVEN

const MyQueue = {
Type: 'AWS::SQS::Queue',
Properties: {},
};
const stack = {
stackName: 'StackWithQueue',
template: {
Resources: {
MyQueue,
CDKMetadata: {
Type: 'AWS::CDK::Metadata',
Properties: {
Analytics: 'exists',
},
},
},
Outputs: {
Output: {
Description: 'There is an output',
Value: 'OutputValue',
},
},
},
};

givenCurrentStack(stack.stackName, stack);
const importer = new ResourceImporter(testStack(stack), deployments);
const migrateMap = [{
LogicalResourceId: 'MyQueue',
ResourceIdentifier: { QueueName: 'TheQueueName' },
ResourceType: 'AWS::SQS::Queue',
}];

// WHEN
await importer.importResourcesFromMigrate(migrateMap, STACK_WITH_QUEUE.template);

// THEN
expect(createChangeSetInput?.ResourcesToImport).toEqual(migrateMap);
expect(createChangeSetInput?.TemplateBody).toEqual('Resources:\n MyQueue:\n Type: AWS::SQS::Queue\n Properties: {}\n');
});

test('only use one identifier if multiple are in template', async () => {
// GIVEN
const stack = stackWithGlobalTable({
Expand Down Expand Up @@ -289,7 +331,7 @@ async function importTemplateFromClean(stack: ReturnType<typeof testStack>) {
const importer = new ResourceImporter(stack, deployments);
const { additions } = await importer.discoverImportableResources();
const importable = await importer.askForResourceIdentifiers(additions);
await importer.importResources(importable, { stack });
await importer.importResourcesFromMap(importable, {});
return importable;
}

Expand Down

0 comments on commit 811ec58

Please sign in to comment.