feat(core): customize bundling output packaging

allowed-breaking-changes.txt

@@ -56,10 +56,3 @@ incompatible-argument:@aws-cdk/aws-ecs.TaskDefinition.addVolume
 # We made properties optional and it's really fine but our differ doesn't think so.
 weakened:@aws-cdk/cloud-assembly-schema.DockerImageSource
 weakened:@aws-cdk/cloud-assembly-schema.FileSource
-
-# https:/aws/aws-cdk/pull/13145
-removed:@aws-cdk/core.AssetStaging.isArchive
-removed:@aws-cdk/core.AssetStaging.packaging
-removed:@aws-cdk/core.BundlingOutput
-removed:@aws-cdk/core.BundlingOptions.outputType
-

packages/@aws-cdk/aws-s3-assets/README.md

@@ -124,6 +124,27 @@ new assets.Asset(this, 'BundledAsset', {
 Although optional, it's recommended to provide a local bundling method which can
 greatly improve performance.
 
+If the bundling output contains a single archive file (zip or jar) it will be
+uploaded to S3 as-is and will not be zipped. Otherwise the contents of the
+output directory will be zipped and the zip file will be uploaded to S3. This
+is the default behavior for `bundling.outputType` (`BundlingOutput.AUTO_DISCOVER`).
+
+Use `BundlingOutput.NOT_ARCHIVED` if the bundling output must always be zipped:
+
+```ts
+const asset = new assets.Asset(this, 'BundledAsset', {
+ path: '/path/to/asset',
+ bundling: {
+ image: BundlingDockerImage.fromRegistry('alpine'),
+ command: ['command-that-produces-an-archive.sh'],
+ outputType: BundlingOutput.NOT_ARCHIVED, // Bundling output will be zipped even though it produces a single archive file.
+ },
+});
+```
+
+Use `BundlingOutput.ARCHIVED` if the bundling output contains a single archive file and
+you don't want it to be zippped.
+
 ## CloudFormation Resource Metadata
 
 > NOTE: This section is relevant for authors of AWS Resource Constructs.

packages/@aws-cdk/aws-s3-assets/lib/asset.ts

@@ -1,4 +1,3 @@
-import * as fs from 'fs';
 import * as path from 'path';
 import * as assets from '@aws-cdk/assets';
 import * as iam from '@aws-cdk/aws-iam';
@@ -13,8 +12,6 @@ import { toSymlinkFollow } from './compat';
 // eslint-disable-next-line no-duplicate-imports, import/order
 import { Construct as CoreConstruct } from '@aws-cdk/core';
 
-const ARCHIVE_EXTENSIONS = ['.zip', '.jar'];
-
 export interface AssetOptions extends assets.CopyOptions, cdk.AssetOptions {
  /**
  * A list of principals that should be able to read this asset from S3.
@@ -139,17 +136,12 @@ export class Asset extends CoreConstruct implements cdk.IAsset {
 
  this.assetPath = staging.relativeStagedPath(stack);
 
- const packaging = determinePackaging(staging.sourcePath);
-
- this.isFile = packaging === cdk.FileAssetPackaging.FILE;
+ this.isFile = staging.packaging === cdk.FileAssetPackaging.FILE;
 
- // sets isZipArchive based on the type of packaging and file extension
- this.isZipArchive = packaging === cdk.FileAssetPackaging.ZIP_DIRECTORY
- ? true
- : ARCHIVE_EXTENSIONS.some(ext => staging.sourcePath.toLowerCase().endsWith(ext));
+ this.isZipArchive = staging.isArchive;
 
  const location = stack.synthesizer.addFileAsset({
- packaging,
+ packaging: staging.packaging,
  sourceHash: this.sourceHash,
  fileName: this.assetPath,
  });
@@ -210,19 +202,3 @@ export class Asset extends CoreConstruct implements cdk.IAsset {
  this.bucket.grantRead(grantee);
  }
 }
-
-function determinePackaging(assetPath: string): cdk.FileAssetPackaging {
- if (!fs.existsSync(assetPath)) {
- throw new Error(`Cannot find asset at ${assetPath}`);
- }
-
- if (fs.statSync(assetPath).isDirectory()) {
- return cdk.FileAssetPackaging.ZIP_DIRECTORY;
- }
-
- if (fs.statSync(assetPath).isFile()) {
- return cdk.FileAssetPackaging.FILE;
- }
-
- throw new Error(`Asset ${assetPath} is expected to be either a directory or a regular file`);
-}

packages/@aws-cdk/core/lib/asset-staging.ts

@@ -5,8 +5,8 @@ import * as cxapi from '@aws-cdk/cx-api';
 import { Construct } from 'constructs';
 import * as fs from 'fs-extra';
 import * as minimatch from 'minimatch';
-import { AssetHashType, AssetOptions } from './assets';
-import { BundlingOptions } from './bundling';
+import { AssetHashType, AssetOptions, FileAssetPackaging } from './assets';
+import { BundlingOptions, BundlingOutput } from './bundling';
 import { FileSystem, FingerprintOptions } from './fs';
 import { Names } from './names';
 import { Cache } from './private/cache';
@@ -17,6 +17,8 @@ import { Stage } from './stage';
 // eslint-disable-next-line
 import { Construct as CoreConstruct } from './construct-compat';
 
+const ARCHIVE_EXTENSIONS = ['.zip', '.jar'];
+
 /**
  * A previously staged asset
  */
@@ -30,6 +32,16 @@ interface StagedAsset {
  * The hash we used previously
  */
  readonly assetHash: string;
+
+ /**
+ * The packaging of the asset
+ */
+ readonly packaging: FileAssetPackaging,
+
+ /**
+ * Whether this asset is an archive
+ */
+ readonly isArchive: boolean;
 }
 
 /**
@@ -124,6 +136,16 @@ export class AssetStaging extends CoreConstruct {
  */
  public readonly assetHash: string;
 
+ /**
+ * How this asset should be packaged.
+ */
+ public readonly packaging: FileAssetPackaging;
+
+ /**
+ * Whether this asset is an archive (zip or jar).
+ */
+ public readonly isArchive: boolean;
+
  private readonly fingerprintOptions: FingerprintOptions;
 
  private readonly hashType: AssetHashType;
@@ -138,12 +160,20 @@ export class AssetStaging extends CoreConstruct {
 
  private readonly cacheKey: string;
 
+ private readonly sourceStats: fs.Stats;
+
  constructor(scope: Construct, id: string, props: AssetStagingProps) {
  super(scope, id);
 
  this.sourcePath = path.resolve(props.sourcePath);
  this.fingerprintOptions = props;
 
+ if (!fs.existsSync(this.sourcePath)) {
+ throw new Error(`Cannot find asset at ${this.sourcePath}`);
+ }
+
+ this.sourceStats = fs.statSync(this.sourcePath);
+
  const outdir = Stage.of(this)?.assetOutdir;
  if (!outdir) {
  throw new Error('unable to determine cloud assembly asset output directory. Assets must be defined indirectly within a "Stage" or an "App" scope');
@@ -192,6 +222,8 @@ export class AssetStaging extends CoreConstruct {
  this.stagedPath = staged.stagedPath;
  this.absoluteStagedPath = staged.stagedPath;
  this.assetHash = staged.assetHash;
+ this.packaging = staged.packaging;
+ this.isArchive = staged.isArchive;
  }
 
  /**
@@ -248,8 +280,18 @@ export class AssetStaging extends CoreConstruct {
  ? this.sourcePath
  : path.resolve(this.assetOutdir, renderAssetFilename(assetHash, path.extname(this.sourcePath)));
 
+ if (!this.sourceStats.isDirectory() && !this.sourceStats.isFile()) {
+ throw new Error(`Asset ${this.sourcePath} is expected to be either a directory or a regular file`);
+ }
+
  this.stageAsset(this.sourcePath, stagedPath, 'copy');
- return { assetHash, stagedPath };
+
+ return {
+ assetHash,
+ stagedPath,
+ packaging: this.sourceStats.isDirectory() ? FileAssetPackaging.ZIP_DIRECTORY : FileAssetPackaging.FILE,
+ isArchive: this.sourceStats.isDirectory() || ARCHIVE_EXTENSIONS.includes(path.extname(this.sourcePath).toLowerCase()),
+ };
  }
 
  /**
@@ -258,6 +300,10 @@ export class AssetStaging extends CoreConstruct {
  * Optionally skip, in which case we pretend we did something but we don't really.
  */
  private stageByBundling(bundling: BundlingOptions, skip: boolean): StagedAsset {
+ if (!this.sourceStats.isDirectory()) {
+ throw new Error(`Asset ${this.sourcePath} is expected to be a directory when bundling`);
+ }
+
  if (skip) {
  // We should have bundled, but didn't to save time. Still pretend to have a hash.
  // If the asset uses OUTPUT or BUNDLE, we use a CUSTOM hash to avoid fingerprinting
@@ -270,6 +316,8 @@ export class AssetStaging extends CoreConstruct {
  return {
  assetHash: this.calculateHash(hashType, bundling),
  stagedPath: this.sourcePath,
+ packaging: FileAssetPackaging.ZIP_DIRECTORY,
+ isArchive: true,
  };
  }
 
@@ -281,12 +329,21 @@ export class AssetStaging extends CoreConstruct {
  const bundleDir = this.determineBundleDir(this.assetOutdir, assetHash);
  this.bundle(bundling, bundleDir);
 
- // Calculate assetHash afterwards if we still must
- assetHash = assetHash ?? this.calculateHash(this.hashType, bundling, bundleDir);
- const stagedPath = path.resolve(this.assetOutdir, renderAssetFilename(assetHash));
+ // Check bundling output content and determine if we will need to archive
+ const bundlingOutputType = bundling.outputType ?? BundlingOutput.AUTO_DISCOVER;
+ const bundledAsset = determineBundledAsset(bundleDir, bundlingOutputType);
 
- this.stageAsset(bundleDir, stagedPath, 'move');
- return { assetHash, stagedPath };
+ // Calculate assetHash afterwards if we still must
+ assetHash = assetHash ?? this.calculateHash(this.hashType, bundling, bundledAsset.path);
+ const stagedPath = path.resolve(this.assetOutdir, renderAssetFilename(assetHash, bundledAsset.extension));
+
+ this.stageAsset(bundledAsset.path, stagedPath, 'move');
+ return {
+ assetHash,
+ stagedPath,
+ packaging: bundledAsset.packaging,
+ isArchive: true, // bundling always produces an archive
+ };
  }
 
  /**
@@ -320,10 +377,9 @@ export class AssetStaging extends CoreConstruct {
  }
 
  // Copy file/directory to staging directory
- const stat = fs.statSync(sourcePath);
- if (stat.isFile()) {
+ if (this.sourceStats.isFile()) {
  fs.copyFileSync(sourcePath, targetPath);
- } else if (stat.isDirectory()) {
+ } else if (this.sourceStats.isDirectory()) {
  fs.mkdirSync(targetPath);
  FileSystem.copyDirectory(sourcePath, targetPath, this.fingerprintOptions);
  } else {
@@ -502,3 +558,57 @@ function sortObject(object: { [key: string]: any }): { [key: string]: any } {
  }
  return ret;
 }
+
+/**
+ * Returns the single archive file of a directory or undefined
+ */
+function singleArchiveFile(directory: string): string | undefined {
+ if (!fs.existsSync(directory)) {
+ throw new Error(`Directory ${directory} does not exist.`);
+ }
+
+ if (!fs.statSync(directory).isDirectory()) {
+ throw new Error(`${directory} is not a directory.`);
+ }
+
+ const content = fs.readdirSync(directory);
+ if (content.length === 1) {
+ const file = path.join(directory, content[0]);
+ const extension = path.extname(content[0]).toLowerCase();
+ if (fs.statSync(file).isFile() && ARCHIVE_EXTENSIONS.includes(extension)) {
+ return file;
+ }
+ }
+
+ return undefined;
+}
+
+interface BundledAsset {
+ path: string,
+ packaging: FileAssetPackaging,
+ extension?: string
+}
+
+/**
+ * Returns the bundled asset to use based on the content of the bundle directory
+ * and the type of output.
+ */
+function determineBundledAsset(bundleDir: string, outputType: BundlingOutput): BundledAsset {
+ const archiveFile = singleArchiveFile(bundleDir);
+
+ // auto-discover means that if there is an archive file, we take it as the
+ // bundle, otherwise, we will archive here.
+ if (outputType === BundlingOutput.AUTO_DISCOVER) {
+ outputType = archiveFile ? BundlingOutput.ARCHIVED : BundlingOutput.NOT_ARCHIVED;
+ }
+
+ switch (outputType) {
+ case BundlingOutput.NOT_ARCHIVED:
+ return { path: bundleDir, packaging: FileAssetPackaging.ZIP_DIRECTORY };
+ case BundlingOutput.ARCHIVED:
+ if (!archiveFile) {
+ throw new Error('Bundling output directory is expected to include only a single .zip or .jar file when `output` is set to `ARCHIVED`');
+ }
+ return { path: archiveFile, packaging: FileAssetPackaging.FILE, extension: path.extname(archiveFile) };
+ }
+}

packages/@aws-cdk/core/lib/bundling.ts

@@ -80,6 +80,41 @@ export interface BundlingOptions {
  * @experimental
  */
  readonly local?: ILocalBundling;
+
+ /**
+ * The type of output that this bundling operation is producing.
+ *
+ * @default BundlingOutput.AUTO_DISCOVER
+ *
+ * @experimental
+ */
+ readonly outputType?: BundlingOutput;
+}
+
+/**
+ * The type of output that a bundling operation is producing.
+ *
+ * @experimental
+ */
+export enum BundlingOutput {
+ /**
+ * The bundling output directory includes a single .zip or .jar file which
+ * will be used as the final bundle. If the output directory does not
+ * include exactly a single archive, bundling will fail.
+ */
+ ARCHIVED = 'archived',
+
+ /**
+ * The bundling output directory contains one or more files which will be
+ * archived and uploaded as a .zip file to S3.
+ */
+ NOT_ARCHIVED = 'not-archived',
+
+ /**
+ * If the bundling output directory contains a single archive file (zip or jar)
+ * it will be used as the bundle output as-is. Otherwise all the files in the bundling output directory will be zipped.
+ */
+ AUTO_DISCOVER = 'auto-discover',
 }
 
 /**

packages/@aws-cdk/core/test/docker-stub.sh

@@ -24,5 +24,18 @@ if echo "$@" | grep "DOCKER_STUB_SUCCESS"; then
  exit 0
 fi
 
-echo "Docker mock only supports one of the following commands: DOCKER_STUB_SUCCESS_NO_OUTPUT,DOCKER_STUB_FAIL,DOCKER_STUB_SUCCESS"
+if echo "$@" | grep "DOCKER_STUB_MULTIPLE_FILES"; then
+ outdir=$(echo "$@" | xargs -n1 | grep "/asset-output" | head -n1 | cut -d":" -f1)
+ touch ${outdir}/test1.txt
+ touch ${outdir}/test2.txt
+ exit 0
+fi
+
+if echo "$@" | grep "DOCKER_STUB_SINGLE_ARCHIVE"; then
+ outdir=$(echo "$@" | xargs -n1 | grep "/asset-output" | head -n1 | cut -d":" -f1)
+ touch ${outdir}/test.zip
+ exit 0
+fi
+
+echo "Docker mock only supports one of the following commands: DOCKER_STUB_SUCCESS_NO_OUTPUT,DOCKER_STUB_FAIL,DOCKER_STUB_SUCCESS,DOCKER_STUB_MULTIPLE_FILES,DOCKER_SINGLE_ARCHIVE"
 exit 1