chore(core): allow the bundler to re-use pre-existing bundler output …

…(revisited) (#9576) This PR changes `AssetStaging` so that the bundler will re-use pre-existing output. Before, the bundler would re-run Docker without considering pre-existing assets, which was slow. Now, when handling a `SOURCE` hash type, the bundler detects and returns pre-existing asset output without re-running Docker. For all other hash types, the bundler outputs to an intermediate directory before calculating asset hashes, then renames the intermediate directory into its final location. This PR revisits #8916 which originally closed #8882. Here are some details from the previous PR which have been addressed in this PR: - The bundler now outputs directly into the assembly directory - The bundler's assets can be reused between multiple syntheses - The bundler keeps output from failed bundling attempts for diagnosability purposes (renamed with an `-error` suffix) - Bundler options are hashed together with custom and source hashes - Removed the check for a docker run from `throws with assetHash and not CUSTOM hash type` as docker is no longer run before the AssetStaging props are validated. ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*
aws · Aug 26, 2020 · f5c9124 · f5c9124
1 parent a4185a0
commit f5c9124
Show file tree

Hide file tree

Showing 10 changed files with 473 additions and 106 deletions.
diff --git a/packages/@aws-cdk/aws-lambda-python/test/integ.function.expected.json b/packages/@aws-cdk/aws-lambda-python/test/integ.function.expected.json
@@ -36,7 +36,7 @@
       "Properties": {
         "Code": {
           "S3Bucket": {
-            "Ref": "AssetParameters822ec544787c04e6b27b02f2408e4c322d48fbd352adb46d5aa2d6b3553a1adeS3Bucket0552B5BB"
+            "Ref": "AssetParametersf7ff2d5c8b3e609d156f7eccbc393419969e89acef6fcc4be4a7091684c1e4d4S3Bucket3D64A262"
           },
           "S3Key": {
             "Fn::Join": [
@@ -49,7 +49,7 @@
                       "Fn::Split": [
                         "||",
                         {
-                          "Ref": "AssetParameters822ec544787c04e6b27b02f2408e4c322d48fbd352adb46d5aa2d6b3553a1adeS3VersionKey9522AC10"
+                          "Ref": "AssetParametersf7ff2d5c8b3e609d156f7eccbc393419969e89acef6fcc4be4a7091684c1e4d4S3VersionKey676CE5F0"
                         }
                       ]
                     }
@@ -62,7 +62,7 @@
                       "Fn::Split": [
                         "||",
                         {
-                          "Ref": "AssetParameters822ec544787c04e6b27b02f2408e4c322d48fbd352adb46d5aa2d6b3553a1adeS3VersionKey9522AC10"
+                          "Ref": "AssetParametersf7ff2d5c8b3e609d156f7eccbc393419969e89acef6fcc4be4a7091684c1e4d4S3VersionKey676CE5F0"
                         }
                       ]
                     }
@@ -87,17 +87,17 @@
     }
   },
   "Parameters": {
-    "AssetParameters822ec544787c04e6b27b02f2408e4c322d48fbd352adb46d5aa2d6b3553a1adeS3Bucket0552B5BB": {
+    "AssetParametersf7ff2d5c8b3e609d156f7eccbc393419969e89acef6fcc4be4a7091684c1e4d4S3Bucket3D64A262": {
       "Type": "String",
-      "Description": "S3 bucket for asset \"822ec544787c04e6b27b02f2408e4c322d48fbd352adb46d5aa2d6b3553a1ade\""
+      "Description": "S3 bucket for asset \"f7ff2d5c8b3e609d156f7eccbc393419969e89acef6fcc4be4a7091684c1e4d4\""
     },
-    "AssetParameters822ec544787c04e6b27b02f2408e4c322d48fbd352adb46d5aa2d6b3553a1adeS3VersionKey9522AC10": {
+    "AssetParametersf7ff2d5c8b3e609d156f7eccbc393419969e89acef6fcc4be4a7091684c1e4d4S3VersionKey676CE5F0": {
       "Type": "String",
-      "Description": "S3 key for asset version \"822ec544787c04e6b27b02f2408e4c322d48fbd352adb46d5aa2d6b3553a1ade\""
+      "Description": "S3 key for asset version \"f7ff2d5c8b3e609d156f7eccbc393419969e89acef6fcc4be4a7091684c1e4d4\""
     },
-    "AssetParameters822ec544787c04e6b27b02f2408e4c322d48fbd352adb46d5aa2d6b3553a1adeArtifactHash3CE06D09": {
+    "AssetParametersf7ff2d5c8b3e609d156f7eccbc393419969e89acef6fcc4be4a7091684c1e4d4ArtifactHashAFB301FE": {
       "Type": "String",
-      "Description": "Artifact hash for asset \"822ec544787c04e6b27b02f2408e4c322d48fbd352adb46d5aa2d6b3553a1ade\""
+      "Description": "Artifact hash for asset \"f7ff2d5c8b3e609d156f7eccbc393419969e89acef6fcc4be4a7091684c1e4d4\""
     }
   },
   "Outputs": {

diff --git a/packages/@aws-cdk/aws-lambda-python/test/integ.function.py38.expected.json b/packages/@aws-cdk/aws-lambda-python/test/integ.function.py38.expected.json
@@ -36,7 +36,7 @@
       "Properties": {
         "Code": {
           "S3Bucket": {
-            "Ref": "AssetParameters822ec544787c04e6b27b02f2408e4c322d48fbd352adb46d5aa2d6b3553a1adeS3Bucket0552B5BB"
+            "Ref": "AssetParameters951d4d5feb891818f169fae0666f355ee76ca00ff218f5b171de713d8b8da576S3Bucket66CABF48"
           },
           "S3Key": {
             "Fn::Join": [
@@ -49,7 +49,7 @@
                       "Fn::Split": [
                         "||",
                         {
-                          "Ref": "AssetParameters822ec544787c04e6b27b02f2408e4c322d48fbd352adb46d5aa2d6b3553a1adeS3VersionKey9522AC10"
+                          "Ref": "AssetParameters951d4d5feb891818f169fae0666f355ee76ca00ff218f5b171de713d8b8da576S3VersionKey46E55A2B"
                         }
                       ]
                     }
@@ -62,7 +62,7 @@
                       "Fn::Split": [
                         "||",
                         {
-                          "Ref": "AssetParameters822ec544787c04e6b27b02f2408e4c322d48fbd352adb46d5aa2d6b3553a1adeS3VersionKey9522AC10"
+                          "Ref": "AssetParameters951d4d5feb891818f169fae0666f355ee76ca00ff218f5b171de713d8b8da576S3VersionKey46E55A2B"
                         }
                       ]
                     }
@@ -87,17 +87,17 @@
     }
   },
   "Parameters": {
-    "AssetParameters822ec544787c04e6b27b02f2408e4c322d48fbd352adb46d5aa2d6b3553a1adeS3Bucket0552B5BB": {
+    "AssetParameters951d4d5feb891818f169fae0666f355ee76ca00ff218f5b171de713d8b8da576S3Bucket66CABF48": {
       "Type": "String",
-      "Description": "S3 bucket for asset \"822ec544787c04e6b27b02f2408e4c322d48fbd352adb46d5aa2d6b3553a1ade\""
+      "Description": "S3 bucket for asset \"951d4d5feb891818f169fae0666f355ee76ca00ff218f5b171de713d8b8da576\""
     },
-    "AssetParameters822ec544787c04e6b27b02f2408e4c322d48fbd352adb46d5aa2d6b3553a1adeS3VersionKey9522AC10": {
+    "AssetParameters951d4d5feb891818f169fae0666f355ee76ca00ff218f5b171de713d8b8da576S3VersionKey46E55A2B": {
       "Type": "String",
-      "Description": "S3 key for asset version \"822ec544787c04e6b27b02f2408e4c322d48fbd352adb46d5aa2d6b3553a1ade\""
+      "Description": "S3 key for asset version \"951d4d5feb891818f169fae0666f355ee76ca00ff218f5b171de713d8b8da576\""
     },
-    "AssetParameters822ec544787c04e6b27b02f2408e4c322d48fbd352adb46d5aa2d6b3553a1adeArtifactHash3CE06D09": {
+    "AssetParameters951d4d5feb891818f169fae0666f355ee76ca00ff218f5b171de713d8b8da576ArtifactHash0B035A28": {
       "Type": "String",
-      "Description": "Artifact hash for asset \"822ec544787c04e6b27b02f2408e4c322d48fbd352adb46d5aa2d6b3553a1ade\""
+      "Description": "Artifact hash for asset \"951d4d5feb891818f169fae0666f355ee76ca00ff218f5b171de713d8b8da576\""
     }
   },
   "Outputs": {

diff --git a/packages/@aws-cdk/aws-lambda-python/test/lambda-handler/index.py b/packages/@aws-cdk/aws-lambda-python/test/lambda-handler/index.py
@@ -2,8 +2,8 @@
 from PIL import Image
 
 def handler(event, context):
-  response = requests.get('https://a0.awsstatic.com/main/images/logos/aws_smile-header-desktop-en-white_59x35.png', stream=True).raw
-  img = Image.open(response)
+  response = requests.get('https://a0.awsstatic.com/main/images/logos/aws_smile-header-desktop-en-white_59x35.png', stream=True)
+  img = Image.open(response.raw)
 
   print(response.status_code)
   print(img.size)

diff --git a/packages/@aws-cdk/aws-lambda/test/integ.bundling.expected.json b/packages/@aws-cdk/aws-lambda/test/integ.bundling.expected.json
@@ -36,7 +36,7 @@
       "Properties": {
         "Code": {
           "S3Bucket": {
-            "Ref": "AssetParameters0ccf37fa0b92d4598d010192eb994040c2e22cc6b12270736d323437817112cdS3Bucket6365D8AA"
+            "Ref": "AssetParametersfbf992a3e922c30f9aed74db034a1ec115ec16544160e6820e5d2463bf1e29d1S3BucketF4EA3D4A"
           },
           "S3Key": {
             "Fn::Join": [
@@ -49,7 +49,7 @@
                       "Fn::Split": [
                         "||",
                         {
-                          "Ref": "AssetParameters0ccf37fa0b92d4598d010192eb994040c2e22cc6b12270736d323437817112cdS3VersionKey14A1DBA7"
+                          "Ref": "AssetParametersfbf992a3e922c30f9aed74db034a1ec115ec16544160e6820e5d2463bf1e29d1S3VersionKey50AB224E"
                         }
                       ]
                     }
@@ -62,7 +62,7 @@
                       "Fn::Split": [
                         "||",
                         {
-                          "Ref": "AssetParameters0ccf37fa0b92d4598d010192eb994040c2e22cc6b12270736d323437817112cdS3VersionKey14A1DBA7"
+                          "Ref": "AssetParametersfbf992a3e922c30f9aed74db034a1ec115ec16544160e6820e5d2463bf1e29d1S3VersionKey50AB224E"
                         }
                       ]
                     }
@@ -87,17 +87,17 @@
     }
   },
   "Parameters": {
-    "AssetParameters0ccf37fa0b92d4598d010192eb994040c2e22cc6b12270736d323437817112cdS3Bucket6365D8AA": {
+    "AssetParametersfbf992a3e922c30f9aed74db034a1ec115ec16544160e6820e5d2463bf1e29d1S3BucketF4EA3D4A": {
       "Type": "String",
-      "Description": "S3 bucket for asset \"0ccf37fa0b92d4598d010192eb994040c2e22cc6b12270736d323437817112cd\""
+      "Description": "S3 bucket for asset \"fbf992a3e922c30f9aed74db034a1ec115ec16544160e6820e5d2463bf1e29d1\""
     },
-    "AssetParameters0ccf37fa0b92d4598d010192eb994040c2e22cc6b12270736d323437817112cdS3VersionKey14A1DBA7": {
+    "AssetParametersfbf992a3e922c30f9aed74db034a1ec115ec16544160e6820e5d2463bf1e29d1S3VersionKey50AB224E": {
       "Type": "String",
-      "Description": "S3 key for asset version \"0ccf37fa0b92d4598d010192eb994040c2e22cc6b12270736d323437817112cd\""
+      "Description": "S3 key for asset version \"fbf992a3e922c30f9aed74db034a1ec115ec16544160e6820e5d2463bf1e29d1\""
     },
-    "AssetParameters0ccf37fa0b92d4598d010192eb994040c2e22cc6b12270736d323437817112cdArtifactHashEEC2ED67": {
+    "AssetParametersfbf992a3e922c30f9aed74db034a1ec115ec16544160e6820e5d2463bf1e29d1ArtifactHashDD1BB80E": {
       "Type": "String",
-      "Description": "Artifact hash for asset \"0ccf37fa0b92d4598d010192eb994040c2e22cc6b12270736d323437817112cd\""
+      "Description": "Artifact hash for asset \"fbf992a3e922c30f9aed74db034a1ec115ec16544160e6820e5d2463bf1e29d1\""
     }
   },
   "Outputs": {

diff --git a/packages/@aws-cdk/aws-s3-assets/test/integ.assets.bundling.lit.expected.json b/packages/@aws-cdk/aws-s3-assets/test/integ.assets.bundling.lit.expected.json
@@ -1,16 +1,16 @@
 {
   "Parameters": {
-    "AssetParameters10af79ff4bd6432db05b51810586c19ca95abd08759fca785e44f594bc9633b8S3Bucket8CD0F73B": {
+    "AssetParameters8995e9405bdcae88dc6fc76b4fc224fecfd00ef93663cb759b491c6a13cc59c2S3Bucket32756583": {
       "Type": "String",
-      "Description": "S3 bucket for asset \"10af79ff4bd6432db05b51810586c19ca95abd08759fca785e44f594bc9633b8\""
+      "Description": "S3 bucket for asset \"8995e9405bdcae88dc6fc76b4fc224fecfd00ef93663cb759b491c6a13cc59c2\""
     },
-    "AssetParameters10af79ff4bd6432db05b51810586c19ca95abd08759fca785e44f594bc9633b8S3VersionKeyA9EAF743": {
+    "AssetParameters8995e9405bdcae88dc6fc76b4fc224fecfd00ef93663cb759b491c6a13cc59c2S3VersionKey39CCFFC8": {
       "Type": "String",
-      "Description": "S3 key for asset version \"10af79ff4bd6432db05b51810586c19ca95abd08759fca785e44f594bc9633b8\""
+      "Description": "S3 key for asset version \"8995e9405bdcae88dc6fc76b4fc224fecfd00ef93663cb759b491c6a13cc59c2\""
     },
-    "AssetParameters10af79ff4bd6432db05b51810586c19ca95abd08759fca785e44f594bc9633b8ArtifactHashBAE492DD": {
+    "AssetParameters8995e9405bdcae88dc6fc76b4fc224fecfd00ef93663cb759b491c6a13cc59c2ArtifactHashDF68D9B9": {
       "Type": "String",
-      "Description": "Artifact hash for asset \"10af79ff4bd6432db05b51810586c19ca95abd08759fca785e44f594bc9633b8\""
+      "Description": "Artifact hash for asset \"8995e9405bdcae88dc6fc76b4fc224fecfd00ef93663cb759b491c6a13cc59c2\""
     }
   },
   "Resources": {
@@ -40,7 +40,7 @@
                       },
                       ":s3:::",
                       {
-                        "Ref": "AssetParameters10af79ff4bd6432db05b51810586c19ca95abd08759fca785e44f594bc9633b8S3Bucket8CD0F73B"
+                        "Ref": "AssetParameters8995e9405bdcae88dc6fc76b4fc224fecfd00ef93663cb759b491c6a13cc59c2S3Bucket32756583"
                       }
                     ]
                   ]
@@ -55,7 +55,7 @@
                       },
                       ":s3:::",
                       {
-                        "Ref": "AssetParameters10af79ff4bd6432db05b51810586c19ca95abd08759fca785e44f594bc9633b8S3Bucket8CD0F73B"
+                        "Ref": "AssetParameters8995e9405bdcae88dc6fc76b4fc224fecfd00ef93663cb759b491c6a13cc59c2S3Bucket32756583"
                       },
                       "/*"
                     ]