From 78b34accfa1cba88cc412b04df42ea5819c2cf4c Mon Sep 17 00:00:00 2001 From: Momo Kornher Date: Thu, 30 Nov 2023 14:57:24 +0100 Subject: [PATCH] fix(cli-lib): deploy fails with "no such file or directory, open 'node_modules/@aws-cdk/integ-runner/lib/workers/db.json.gz'" (#28199) After #27813 the `deploy` action was broken with the above error. This is effectively the same as #27983 . To ensure these kind of issues are not slipping through again, the PR is adding a basic testing harness for `cli-lib` to `@aws-cdk-testing/cli-integtests`. ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license* --- .../@aws-cdk-testing/cli-integ/lib/index.ts | 1 + .../cli-integ/lib/with-cli-lib.ts | 134 ++++++++++++++++++ .../resources/cdk-apps/simple-app/app.js | 26 ++++ .../resources/cdk-apps/simple-app/cdk.json | 7 + .../cli-integ-tests/cli-lib.integtest.ts | 65 +++++++++ packages/@aws-cdk/cli-lib-alpha/.gitignore | 1 + packages/@aws-cdk/cli-lib-alpha/.npmignore | 1 + .../cli-lib-alpha/THIRD_PARTY_LICENSES | 6 +- packages/@aws-cdk/cli-lib-alpha/package.json | 1 + 9 files changed, 239 insertions(+), 3 deletions(-) create mode 100644 packages/@aws-cdk-testing/cli-integ/lib/with-cli-lib.ts create mode 100755 packages/@aws-cdk-testing/cli-integ/resources/cdk-apps/simple-app/app.js create mode 100644 packages/@aws-cdk-testing/cli-integ/resources/cdk-apps/simple-app/cdk.json create mode 100644 packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli-lib.integtest.ts diff --git a/packages/@aws-cdk-testing/cli-integ/lib/index.ts b/packages/@aws-cdk-testing/cli-integ/lib/index.ts index 32ab6ebcb83b4..cabb094e65c00 100644 --- a/packages/@aws-cdk-testing/cli-integ/lib/index.ts +++ b/packages/@aws-cdk-testing/cli-integ/lib/index.ts @@ -3,6 +3,7 @@ export * from './corking'; export * from './integ-test'; export * from './memoize'; export * from './resource-pool'; +export * from './with-cli-lib'; export * from './with-sam'; export * from './shell'; export * from './with-aws'; diff --git a/packages/@aws-cdk-testing/cli-integ/lib/with-cli-lib.ts b/packages/@aws-cdk-testing/cli-integ/lib/with-cli-lib.ts new file mode 100644 index 0000000000000..6ef951ec0c7e3 --- /dev/null +++ b/packages/@aws-cdk-testing/cli-integ/lib/with-cli-lib.ts @@ -0,0 +1,134 @@ +import * as os from 'os'; +import * as path from 'path'; +import { TestContext } from './integ-test'; +import { RESOURCES_DIR } from './resources'; +import { AwsContext, withAws } from './with-aws'; +import { cloneDirectory, installNpmPackages, TestFixture, DEFAULT_TEST_TIMEOUT_S, CdkCliOptions } from './with-cdk-app'; +import { withTimeout } from './with-timeout'; + +/** + * Higher order function to execute a block with a CliLib Integration CDK app fixture + */ +export function withCliLibIntegrationCdkApp(block: (context: CliLibIntegrationTestFixture) => Promise) { + return async (context: A) => { + const randy = context.randomString; + const stackNamePrefix = `cdktest-${randy}`; + const integTestDir = path.join(os.tmpdir(), `cdk-integ-${randy}`); + + context.log(` Stack prefix: ${stackNamePrefix}\n`); + context.log(` Test directory: ${integTestDir}\n`); + context.log(` Region: ${context.aws.region}\n`); + + await cloneDirectory(path.join(RESOURCES_DIR, 'cdk-apps', 'simple-app'), integTestDir, context.output); + const fixture = new CliLibIntegrationTestFixture( + integTestDir, + stackNamePrefix, + context.output, + context.aws, + context.randomString); + + let success = true; + try { + const installationVersion = fixture.packages.requestedFrameworkVersion(); + + if (fixture.packages.majorVersion() === '1') { + throw new Error('This test suite is only compatible with AWS CDK v2'); + } + + const alphaInstallationVersion = fixture.packages.requestedAlphaVersion(); + await installNpmPackages(fixture, { + 'aws-cdk-lib': installationVersion, + '@aws-cdk/cli-lib-alpha': alphaInstallationVersion, + '@aws-cdk/aws-lambda-go-alpha': alphaInstallationVersion, + '@aws-cdk/aws-lambda-python-alpha': alphaInstallationVersion, + 'constructs': '^10', + }); + + await block(fixture); + } catch (e: any) { + // We survive certain cases involving gopkg.in + if (errorCausedByGoPkg(e.message)) { + return; + } + success = false; + throw e; + } finally { + if (process.env.INTEG_NO_CLEAN) { + context.log(`Left test directory in '${integTestDir}' ($INTEG_NO_CLEAN)\n`); + } else { + await fixture.dispose(success); + } + } + }; +} + +/** + * Return whether or not the error is being caused by gopkg.in being down + * + * Our Go build depends on https://gopkg.in/, which has errors pretty often + * (every couple of days). It is run by a single volunteer. + */ +function errorCausedByGoPkg(error: string) { + // The error is different depending on what request fails. Messages recognized: + //////////////////////////////////////////////////////////////////// + // go: github.com/aws/aws-lambda-go@v1.28.0 requires + // gopkg.in/yaml.v3@v3.0.0-20200615113413-eeeca48fe776: invalid version: git ls-remote -q origin in /go/pkg/mod/cache/vcs/0901dc1ef67fcce1c9b3ae51078740de4a0e2dc673e720584ac302973af82f36: exit status 128: + // remote: Cannot obtain refs from GitHub: cannot talk to GitHub: Get https://github.com/go-yaml/yaml.git/info/refs?service=git-upload-pack: net/http: request canceled (Client.Timeout exceeded while awaiting headers) + // fatal: unable to access 'https://gopkg.in/yaml.v3/': The requested URL returned error: 502 + //////////////////////////////////////////////////////////////////// + // go: downloading github.com/aws/aws-lambda-go v1.28.0 + // go: github.com/aws/aws-lambda-go@v1.28.0 requires + // gopkg.in/yaml.v3@v3.0.0-20200615113413-eeeca48fe776: unrecognized import path "gopkg.in/yaml.v3": reading https://gopkg.in/yaml.v3?go-get=1: 502 Bad Gateway + // server response: Cannot obtain refs from GitHub: cannot talk to GitHub: Get https://github.com/go-yaml/yaml.git/info/refs?service=git-upload-pack: net/http: request canceled (Client.Timeout exceeded while awaiting headers) + //////////////////////////////////////////////////////////////////// + // go: github.com/aws/aws-lambda-go@v1.28.0 requires + // gopkg.in/yaml.v3@v3.0.0-20200615113413-eeeca48fe776: invalid version: git fetch -f origin refs/heads/*:refs/heads/* refs/tags/*:refs/tags/* in /go/pkg/mod/cache/vcs/0901dc1ef67fcce1c9b3ae51078740de4a0e2dc673e720584ac302973af82f36: exit status 128: + // error: RPC failed; HTTP 502 curl 22 The requested URL returned error: 502 + // fatal: the remote end hung up unexpectedly + //////////////////////////////////////////////////////////////////// + + return (error.includes('gopkg\.in.*invalid version.*exit status 128') + || error.match(/unrecognized import path[^\n]gopkg\.in/)); +} + +/** + * SAM Integration test fixture for CDK - SAM integration test cases + */ +export function withCliLibFixture(block: (context: CliLibIntegrationTestFixture) => Promise) { + return withAws(withTimeout(DEFAULT_TEST_TIMEOUT_S, withCliLibIntegrationCdkApp(block))); +} + +export class CliLibIntegrationTestFixture extends TestFixture { + /** + * + */ + public async cdk(args: string[], options: CdkCliOptions = {}) { + const action = args[0]; + const stackName = args[1]; + + const cliOpts: Record = { + stacks: stackName ? [stackName] : undefined, + }; + + if (action === 'deploy') { + cliOpts.requireApproval = options.neverRequireApproval ? 'never' : 'broadening'; + } + + return this.shell(['node', '--input-type=module', `<<__EOS__ + import { AwsCdkCli } from '@aws-cdk/cli-lib-alpha'; + const cli = AwsCdkCli.fromCdkAppDirectory(); + + await cli.${action}(${JSON.stringify(cliOpts)}); +__EOS__`], { + ...options, + modEnv: { + AWS_REGION: this.aws.region, + AWS_DEFAULT_REGION: this.aws.region, + STACK_NAME_PREFIX: this.stackNamePrefix, + PACKAGE_LAYOUT_VERSION: this.packages.majorVersion(), + ...options.modEnv, + }, + }); + } + +} diff --git a/packages/@aws-cdk-testing/cli-integ/resources/cdk-apps/simple-app/app.js b/packages/@aws-cdk-testing/cli-integ/resources/cdk-apps/simple-app/app.js new file mode 100755 index 0000000000000..e1c644cdbd4c3 --- /dev/null +++ b/packages/@aws-cdk-testing/cli-integ/resources/cdk-apps/simple-app/app.js @@ -0,0 +1,26 @@ +const cdk = require('aws-cdk-lib/core'); +const iam = require('aws-cdk-lib/aws-iam'); +const sqs = require('aws-cdk-lib/aws-sqs'); + +const stackPrefix = process.env.STACK_NAME_PREFIX; +if (!stackPrefix) { + throw new Error(`the STACK_NAME_PREFIX environment variable is required`); +} + +class SimpleStack extends cdk.Stack { + constructor(scope, id, props) { + super(scope, id, props); + const queue = new sqs.Queue(this, 'queue', { + visibilityTimeout: cdk.Duration.seconds(300), + }); + const role = new iam.Role(this, 'role', { + assumedBy: new iam.AnyPrincipal(), + }); + queue.grantConsumeMessages(role); + } +} + +const app = new cdk.App(); +new SimpleStack(app, `${stackPrefix}-simple-1`); + +app.synth(); diff --git a/packages/@aws-cdk-testing/cli-integ/resources/cdk-apps/simple-app/cdk.json b/packages/@aws-cdk-testing/cli-integ/resources/cdk-apps/simple-app/cdk.json new file mode 100644 index 0000000000000..44809158dbdac --- /dev/null +++ b/packages/@aws-cdk-testing/cli-integ/resources/cdk-apps/simple-app/cdk.json @@ -0,0 +1,7 @@ +{ + "app": "node app.js", + "versionReporting": false, + "context": { + "aws-cdk:enableDiffNoFail": "true" + } +} diff --git a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli-lib.integtest.ts b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli-lib.integtest.ts new file mode 100644 index 0000000000000..2d0da77b48f63 --- /dev/null +++ b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli-lib.integtest.ts @@ -0,0 +1,65 @@ +import { integTest, withCliLibFixture } from '../../lib'; + +jest.setTimeout(2 * 60 * 60_000); // Includes the time to acquire locks, worst-case single-threaded runtime + +integTest('cli-lib synth', withCliLibFixture(async (fixture) => { + await fixture.cdk(['synth', fixture.fullStackName('simple-1')]); + expect(fixture.template('simple-1')).toEqual(expect.objectContaining({ + // Checking for a small subset is enough as proof that synth worked + Resources: expect.objectContaining({ + queue276F7297: expect.objectContaining({ + Type: 'AWS::SQS::Queue', + Properties: { + VisibilityTimeout: 300, + }, + Metadata: { + 'aws:cdk:path': `${fixture.stackNamePrefix}-simple-1/queue/Resource`, + }, + }), + }), + })); +})); + +integTest('cli-lib list', withCliLibFixture(async (fixture) => { + const listing = await fixture.cdk(['list'], { captureStderr: false }); + expect(listing).toContain(fixture.fullStackName('simple-1')); +})); + +integTest('cli-lib deploy', withCliLibFixture(async (fixture) => { + const stackName = fixture.fullStackName('simple-1'); + + try { + // deploy the stack + await fixture.cdk(['deploy', stackName], { + neverRequireApproval: true, + }); + + // verify the number of resources in the stack + const expectedStack = await fixture.aws.cloudFormation('describeStackResources', { + StackName: stackName, + }); + expect(expectedStack.StackResources?.length).toEqual(3); + } finally { + // delete the stack + await fixture.cdk(['destroy', stackName], { + captureStderr: false, + }); + } +})); + +integTest('security related changes without a CLI are expected to fail when approval is required', withCliLibFixture(async (fixture) => { + const stdErr = await fixture.cdk(['deploy', fixture.fullStackName('simple-1')], { + onlyStderr: true, + captureStderr: true, + allowErrExit: true, + neverRequireApproval: false, + }); + + expect(stdErr).toContain('This deployment will make potentially sensitive changes according to your current security approval level'); + expect(stdErr).toContain('Deployment failed: Error: \"--require-approval\" is enabled and stack includes security-sensitive updates'); + + // Ensure stack was not deployed + await expect(fixture.aws.cloudFormation('describeStacks', { + StackName: fixture.fullStackName('simple-1'), + })).rejects.toThrow('does not exist'); +})); diff --git a/packages/@aws-cdk/cli-lib-alpha/.gitignore b/packages/@aws-cdk/cli-lib-alpha/.gitignore index de6b6804a7733..573a12d965554 100644 --- a/packages/@aws-cdk/cli-lib-alpha/.gitignore +++ b/packages/@aws-cdk/cli-lib-alpha/.gitignore @@ -1,6 +1,7 @@ *.js *.js.map *.d.ts +*.gz !lib/init-templates/**/javascript/**/* node_modules dist diff --git a/packages/@aws-cdk/cli-lib-alpha/.npmignore b/packages/@aws-cdk/cli-lib-alpha/.npmignore index 4cd6d7c4a2b6c..a3afe6f7b23d6 100644 --- a/packages/@aws-cdk/cli-lib-alpha/.npmignore +++ b/packages/@aws-cdk/cli-lib-alpha/.npmignore @@ -35,3 +35,4 @@ tsconfig.json !build-info.json !NOTICE !THIRD_PARTY_LICENSES +!db.json.gz diff --git a/packages/@aws-cdk/cli-lib-alpha/THIRD_PARTY_LICENSES b/packages/@aws-cdk/cli-lib-alpha/THIRD_PARTY_LICENSES index 4fc8eaf268c73..ad862cd67eb28 100644 --- a/packages/@aws-cdk/cli-lib-alpha/THIRD_PARTY_LICENSES +++ b/packages/@aws-cdk/cli-lib-alpha/THIRD_PARTY_LICENSES @@ -207,7 +207,7 @@ The @aws-cdk/cli-lib-alpha package includes the following third-party software/l ---------------- -** @jsii/check-node@1.91.0 - https://www.npmjs.com/package/@jsii/check-node/v/1.91.0 | Apache-2.0 +** @jsii/check-node@1.92.0 - https://www.npmjs.com/package/@jsii/check-node/v/1.92.0 | Apache-2.0 jsii Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. @@ -471,7 +471,7 @@ THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH RE ---------------- -** aws-sdk@2.1492.0 - https://www.npmjs.com/package/aws-sdk/v/2.1492.0 | Apache-2.0 +** aws-sdk@2.1498.0 - https://www.npmjs.com/package/aws-sdk/v/2.1498.0 | Apache-2.0 AWS SDK for JavaScript Copyright 2012-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. @@ -668,7 +668,7 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI ---------------- -** cdk-from-cfn@0.67.0 - https://www.npmjs.com/package/cdk-from-cfn/v/0.67.0 | MIT OR Apache-2.0 +** cdk-from-cfn@0.69.0 - https://www.npmjs.com/package/cdk-from-cfn/v/0.69.0 | MIT OR Apache-2.0 ---------------- diff --git a/packages/@aws-cdk/cli-lib-alpha/package.json b/packages/@aws-cdk/cli-lib-alpha/package.json index 3e8212c9e92db..206625baed7f1 100644 --- a/packages/@aws-cdk/cli-lib-alpha/package.json +++ b/packages/@aws-cdk/cli-lib-alpha/package.json @@ -65,6 +65,7 @@ "yarn attribute", "mkdir -p ./lib/api/bootstrap/ && cp ../../aws-cdk/lib/api/bootstrap/bootstrap-template.yaml ./lib/api/bootstrap/", "cp ../../../node_modules/cdk-from-cfn/index_bg.wasm ./lib/", + "cp ../../../node_modules/@aws-cdk/aws-service-spec/db.json.gz ./", "yarn bundle", "node ./lib/main.js >/dev/null 2>/dev/null