feat(s3-deployment): extract flag to disable automatic unzipping (aws…

…#21805)

Currently S3 Deployment will unzip all files when deployed to S3. This PR adds an additional optional prop `extract`, that when set to false will allow files to remain zipped when deployed to S3.

Adds the feature described in issue: aws#8065 

----

### All Submissions:

* [x] Have you followed the guidelines in our [Contributing guide?](https://github.com/aws/aws-cdk/blob/main/CONTRIBUTING.md)

### Adding new Unconventional Dependencies:

* [ ] This PR adds new unconventional dependencies following the process described [here](https://github.com/aws/aws-cdk/blob/main/CONTRIBUTING.md/#adding-new-unconventional-dependencies)

### New Features

* [x] Have you added the new feature to an [integration test](https://github.com/aws/aws-cdk/blob/main/INTEGRATION_TESTS.md)?
	* [x] Did you use `yarn integ` to deploy the infrastructure and generate the snapshot (i.e. `yarn integ` without `--dry-run`)?

*By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*

----
Co-authored-by: Alex Makoviecki [[email protected]](mailto:[email protected])

packages/@aws-cdk/aws-s3-deployment/README.md

@@ -325,6 +325,28 @@ The value in `topic.topicArn` is a deploy-time value. It only gets resolved
 during deployment by placing a marker in the generated source file and
 substituting it when its deployed to the destination with the actual value.
 
+## Keep Files Zipped
+
+By default, files are zipped, then extracted into the destination bucket.
+
+You can use the option `extract: false` to disable this behavior, in which case, files will remain in a zip file when deployed to S3. To reference the object keys, or filenames, which will be deployed to the bucket, you can use the `objectKeys` getter on the bucket deployment.
+
+```ts
+import * as cdk from 'aws-cdk-lib';
+
+declare const destinationBucket: s3.Bucket;
+
+const myBucketDeployment = new s3deploy.BucketDeployment(this, 'DeployMeWithoutExtractingFilesOnDestination', {
+  sources: [s3deploy.Source.asset(path.join(__dirname, 'my-website'))],
+  destinationBucket,
+  extract: false,
+});
+
+new cdk.CfnOutput(this, 'ObjectKey', {
+  value: cdk.Fn.select(0, myBucketDeployment.objectKeys),
+});
+```
+
 ## Notes
 
 - This library uses an AWS CloudFormation custom resource which is about 10MiB in

packages/@aws-cdk/aws-s3-deployment/lib/bucket-deployment.ts

@@ -38,6 +38,13 @@ export interface BucketDeploymentProps {
    */
   readonly destinationKeyPrefix?: string;
 
+  /**
+   * If this is set, the zip file will be synced to the destination S3 bucket and extracted.
+   * If false, the file will remain zipped in the destination bucket.
+   * @default true
+   */
+  readonly extract?: boolean;
+
   /**
    * If this is set, matching files or objects will be excluded from the deployment's sync
    * command. This can be used to exclude a file from being pruned in the destination bucket.
@@ -339,6 +346,12 @@ export class BucketDeployment extends Construct {
     // the sources actually has markers.
     const hasMarkers = sources.some(source => source.markers);
 
+    // Markers are not replaced if zip sources are not extracted, so throw an error
+    // if extraction is not wanted and sources have markers.
+    if (hasMarkers && props.extract == false) {
+      throw new Error('Some sources are incompatible with extract=false; sources with deploy-time values (such as \'snsTopic.topicArn\') must be extracted.');
+    }
+
     const crUniqueId = `CustomResource${this.renderUniqueId(props.memoryLimit, props.ephemeralStorageSize, props.vpc)}`;
     this.cr = new cdk.CustomResource(this, crUniqueId, {
       serviceToken: handler.functionArn,
@@ -350,6 +363,7 @@ export class BucketDeployment extends Construct {
         DestinationBucketName: props.destinationBucket.bucketName,
         DestinationBucketKeyPrefix: props.destinationKeyPrefix,
         RetainOnDelete: props.retainOnDelete,
+        Extract: props.extract,
         Prune: props.prune ?? true,
         Exclude: props.exclude,
         Include: props.include,
@@ -434,6 +448,23 @@ export class BucketDeployment extends Construct {
     return this._deployedBucket;
   }
 
+  /**
+   * The object keys for the sources deployed to the S3 bucket.
+   *
+   * This returns a list of tokenized object keys for source files that are deployed to the bucket.
+   *
+   * This can be useful when using `BucketDeployment` with `extract` set to `false` and you need to reference
+   * the object key that resides in the bucket for that zip source file somewhere else in your CDK
+   * application, such as in a CFN output.
+   *
+   * For example, use `Fn.select(0, myBucketDeployment.objectKeys)` to reference the object key of the
+   * first source file in your bucket deployment.
+   */
+  public get objectKeys(): string[] {
+    const objectKeys = cdk.Token.asList(this.cr.getAtt('SourceObjectKeys'));
+    return objectKeys;
+  }
+
   private renderUniqueId(memoryLimit?: number, ephemeralStorageSize?: cdk.Size, vpc?: ec2.IVpc) {
     let uuid = '';
 

packages/@aws-cdk/aws-s3-deployment/lib/lambda/index.py

@@ -49,6 +49,7 @@ def cfn_error(message=None):
             source_markers      = props.get('SourceMarkers', None)
             dest_bucket_name    = props['DestinationBucketName']
             dest_bucket_prefix  = props.get('DestinationBucketKeyPrefix', '')
+            extract             = props.get('Extract', 'true') == 'true'
             retain_on_delete    = props.get('RetainOnDelete', "true") == "true"
             distribution_id     = props.get('DistributionId', '')
             user_metadata       = props.get('UserMetadata', {})
@@ -113,14 +114,15 @@ def cfn_error(message=None):
             aws_command("s3", "rm", old_s3_dest, "--recursive")
 
         if request_type == "Update" or request_type == "Create":
-            s3_deploy(s3_source_zips, s3_dest, user_metadata, system_metadata, prune, exclude, include, source_markers)
+            s3_deploy(s3_source_zips, s3_dest, user_metadata, system_metadata, prune, exclude, include, source_markers, extract)
 
         if distribution_id:
             cloudfront_invalidate(distribution_id, distribution_paths)
 
         cfn_send(event, context, CFN_SUCCESS, physicalResourceId=physical_id, responseData={
             # Passing through the ARN sequences dependencees on the deployment
-            'DestinationBucketArn': props.get('DestinationBucketArn')
+            'DestinationBucketArn': props.get('DestinationBucketArn'),
+            'SourceObjectKeys': props.get('SourceObjectKeys'),
         })
     except KeyError as e:
         cfn_error("invalid request. Missing key %s" % str(e))
@@ -130,7 +132,7 @@ def cfn_error(message=None):
 
 #---------------------------------------------------------------------------------------------------
 # populate all files from s3_source_zips to a destination bucket
-def s3_deploy(s3_source_zips, s3_dest, user_metadata, system_metadata, prune, exclude, include, source_markers):
+def s3_deploy(s3_source_zips, s3_dest, user_metadata, system_metadata, prune, exclude, include, source_markers, extract):
     # list lengths are equal
     if len(s3_source_zips) != len(source_markers):
         raise Exception("'source_markers' and 's3_source_zips' must be the same length")
@@ -154,12 +156,16 @@ def s3_deploy(s3_source_zips, s3_dest, user_metadata, system_metadata, prune, ex
             s3_source_zip = s3_source_zips[i]
             markers       = source_markers[i]
 
-            archive=os.path.join(workdir, str(uuid4()))
-            logger.info("archive: %s" % archive)
-            aws_command("s3", "cp", s3_source_zip, archive)
-            logger.info("| extracting archive to: %s\n" % contents_dir)
-            logger.info("| markers: %s" % markers)
-            extract_and_replace_markers(archive, contents_dir, markers)
+            if extract:
+                archive=os.path.join(workdir, str(uuid4()))
+                logger.info("archive: %s" % archive)
+                aws_command("s3", "cp", s3_source_zip, archive)
+                logger.info("| extracting archive to: %s\n" % contents_dir)
+                logger.info("| markers: %s" % markers)
+                extract_and_replace_markers(archive, contents_dir, markers)
+            else:
+                logger.info("| copying archive to: %s\n" % contents_dir)
+                aws_command("s3", "cp", s3_source_zip, contents_dir)
 
         # sync from "contents" to destination
 
@@ -285,7 +291,7 @@ def extract_and_replace_markers(archive, contents_dir, markers):
         for file in zip.namelist():
             file_path = os.path.join(contents_dir, file)
             if os.path.isdir(file_path): continue
-            replace_markers(file_path, markers)    
+            replace_markers(file_path, markers)
 
 def replace_markers(filename, markers):
     # convert the dict of string markers to binary markers

packages/@aws-cdk/aws-s3-deployment/package.json

@@ -104,6 +104,7 @@
     "@aws-cdk/aws-logs": "0.0.0",
     "@aws-cdk/aws-s3": "0.0.0",
     "@aws-cdk/aws-s3-assets": "0.0.0",
+    "@aws-cdk/aws-sns": "0.0.0",
     "@aws-cdk/core": "0.0.0",
     "@aws-cdk/lambda-layer-awscli": "0.0.0",
     "case": "1.6.3",
@@ -119,6 +120,7 @@
     "@aws-cdk/aws-logs": "0.0.0",
     "@aws-cdk/aws-s3": "0.0.0",
     "@aws-cdk/aws-s3-assets": "0.0.0",
+    "@aws-cdk/aws-sns": "0.0.0",
     "@aws-cdk/core": "0.0.0",
     "@aws-cdk/lambda-layer-awscli": "0.0.0",
     "constructs": "^10.0.0"

packages/@aws-cdk/aws-s3-deployment/test/bucket-deployment-data.integ.snapshot/cdk.out

@@ -1 +1 @@
-{"version":"20.0.0"}
+{"version":"21.0.0"}

packages/@aws-cdk/aws-s3-deployment/test/bucket-deployment-data.integ.snapshot/integ.json

@@ -1,5 +1,5 @@
 {
-  "version": "20.0.0",
+  "version": "21.0.0",
   "testCases": {
     "integ.bucket-deployment-data": {
       "stacks": [

packages/@aws-cdk/aws-s3-deployment/test/bucket-deployment.integ.snapshot/asset.f98b78092dcdd31f5e6d47489beb5f804d4835ef86a8085d0a2053cb9ae711da/index.py → packages/@aws-cdk/aws-s3-deployment/test/bucket-deployment.integ.snapshot/asset.6ddcf10002539818a9256eff3fb2b22aa09298d8f946e26ba121c175a600c44e/index.py

@@ -49,6 +49,7 @@ def cfn_error(message=None):
             source_markers      = props.get('SourceMarkers', None)
             dest_bucket_name    = props['DestinationBucketName']
             dest_bucket_prefix  = props.get('DestinationBucketKeyPrefix', '')
+            extract             = props.get('Extract', 'true') == 'true'
             retain_on_delete    = props.get('RetainOnDelete', "true") == "true"
             distribution_id     = props.get('DistributionId', '')
             user_metadata       = props.get('UserMetadata', {})
@@ -113,14 +114,15 @@ def cfn_error(message=None):
             aws_command("s3", "rm", old_s3_dest, "--recursive")
 
         if request_type == "Update" or request_type == "Create":
-            s3_deploy(s3_source_zips, s3_dest, user_metadata, system_metadata, prune, exclude, include, source_markers)
+            s3_deploy(s3_source_zips, s3_dest, user_metadata, system_metadata, prune, exclude, include, source_markers, extract)
 
         if distribution_id:
             cloudfront_invalidate(distribution_id, distribution_paths)
 
         cfn_send(event, context, CFN_SUCCESS, physicalResourceId=physical_id, responseData={
             # Passing through the ARN sequences dependencees on the deployment
-            'DestinationBucketArn': props.get('DestinationBucketArn')
+            'DestinationBucketArn': props.get('DestinationBucketArn'),
+            'SourceObjectKeys': props.get('SourceObjectKeys'),
         })
     except KeyError as e:
         cfn_error("invalid request. Missing key %s" % str(e))
@@ -130,7 +132,7 @@ def cfn_error(message=None):
 
 #---------------------------------------------------------------------------------------------------
 # populate all files from s3_source_zips to a destination bucket
-def s3_deploy(s3_source_zips, s3_dest, user_metadata, system_metadata, prune, exclude, include, source_markers):
+def s3_deploy(s3_source_zips, s3_dest, user_metadata, system_metadata, prune, exclude, include, source_markers, extract):
     # list lengths are equal
     if len(s3_source_zips) != len(source_markers):
         raise Exception("'source_markers' and 's3_source_zips' must be the same length")
@@ -154,12 +156,16 @@ def s3_deploy(s3_source_zips, s3_dest, user_metadata, system_metadata, prune, ex
             s3_source_zip = s3_source_zips[i]
             markers       = source_markers[i]
 
-            archive=os.path.join(workdir, str(uuid4()))
-            logger.info("archive: %s" % archive)
-            aws_command("s3", "cp", s3_source_zip, archive)
-            logger.info("| extracting archive to: %s\n" % contents_dir)
-            logger.info("| markers: %s" % markers)
-            extract_and_replace_markers(archive, contents_dir, markers)
+            if extract:
+                archive=os.path.join(workdir, str(uuid4()))
+                logger.info("archive: %s" % archive)
+                aws_command("s3", "cp", s3_source_zip, archive)
+                logger.info("| extracting archive to: %s\n" % contents_dir)
+                logger.info("| markers: %s" % markers)
+                extract_and_replace_markers(archive, contents_dir, markers)
+            else:
+                logger.info("| copying archive to: %s\n" % contents_dir)
+                aws_command("s3", "cp", s3_source_zip, contents_dir)
 
         # sync from "contents" to destination
 
@@ -285,7 +291,7 @@ def extract_and_replace_markers(archive, contents_dir, markers):
         for file in zip.namelist():
             file_path = os.path.join(contents_dir, file)
             if os.path.isdir(file_path): continue
-            replace_markers(file_path, markers)    
+            replace_markers(file_path, markers)
 
 def replace_markers(filename, markers):
     # convert the dict of string markers to binary markers