fix(toolkit): avoid EMFILE and preserve mode when zipping (#3428)

To preserve file order using `archiver` files must be appended serially either using stream or
buffer (appending by file path does not preserve order even when done serially).

Appending using buffer seems to be the only way to solve `EMFILE` errors.

Call `fs.stat` before appending to preserve mode.

Closes #3145, Closes #3344, Closes #3413

packages/aws-cdk/lib/archive.ts

@@ -5,7 +5,7 @@ import glob = require('glob');
 import path = require('path');
 
 export function zipDirectory(directory: string, outputFile: string): Promise<void> {
-  return new Promise((ok, fail) => {
+  return new Promise(async (ok, fail) => {
     // The below options are needed to support following symlinks when building zip files:
     // - nodir: This will prevent symlinks themselves from being copied into the zip.
     // - follow: This will follow symlinks and copy the files within.
@@ -24,12 +24,16 @@ export function zipDirectory(directory: string, outputFile: string): Promise<voi
     archive.on('error', fail);
     archive.pipe(output);
 
-    files.forEach(file => { // Append files serially to ensure file order
-      archive.append(fs.createReadStream(path.join(directory, file)), {
+    // Append files serially to ensure file order
+    for (const file of files) {
+      const fullPath = path.join(directory, file);
+      const [data, stat] = await Promise.all([fs.readFile(fullPath), fs.stat(fullPath)]);
+      archive.append(data, {
         name: file,
         date: new Date('1980-01-01T00:00:00.000Z'), // reset dates to get the same hash for the same content
+        mode: stat.mode,
       });
-    });
+    }
 
     archive.finalize();
 

packages/aws-cdk/test/test.archive.ts

@@ -32,6 +32,12 @@ export = {
     test.equal(dates[0], '1980-01-01T00:00:00.000Z', 'Dates are not reset');
     test.equal(new Set(dates).size, 1, 'Dates are not equal');
 
+    // check that mode is preserved
+    const stat = await fs.stat(path.join(extractDir, 'executable.txt'));
+    // tslint:disable-next-line:no-bitwise
+    const isExec = (stat.mode & fs.constants.S_IXUSR) || (stat.mode & fs.constants.S_IXGRP) || (stat.mode & fs.constants.S_IXOTH);
+    test.ok(isExec, 'File is not executable');
+
     await fs.remove(stagingDir);
     await fs.remove(extractDir);
     test.done();