feat: CDK Build Integration Test

Adds @jsii/integ-test private module for defining new integration tests.
Adds a new integration test that downloads the latest CDK release source
code and builds it with the local version of jsii and jsii-pacmak.

This unit test requires a github access token defined in the environment
to get the latest release version and download the asset.

Fixes: aws#1209

package.json

@@ -7,6 +7,7 @@
     "fetch-dotnet-snk": "bash scripts/fetch-dotnet-snk.sh",
     "package": "bash scripts/package.sh",
     "test": "lerna run test --stream",
+    "test:integ": "lerna run test:integ --stream",
     "test:update": "lerna run test:update --stream"
   },
   "devDependencies": {

packages/@jsii/integ-test/.env.example

@@ -0,0 +1 @@
+GITHUB_TOKEN=personal_access_token

packages/@jsii/integ-test/.gitignore

@@ -0,0 +1,2 @@
+.env
+*.js

packages/@jsii/integ-test/README.md

@@ -0,0 +1,10 @@
+# JSII Integration Tests
+
+A suite of integration tests for JSII and related modules.
+
+## Running
+
+Running the integration tests locally requires a github access token. Copy the
+.env.example file and replace the dummy value with a personal access token.
+
+then run `yarn run test:integ`

packages/@jsii/integ-test/package.json

@@ -0,0 +1,29 @@
+{
+  "name": "integ-test",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "private": true,
+  "scripts": {
+    "build": "tsc",
+    "test:integ": "jest"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "@octokit/rest": "^16.36.0",
+    "dotenv": "^8.2.0",
+    "jest": "^25.1.0",
+    "jsii": "^0.21.2",
+    "jsii-pacmak": "^0.21.2",
+    "typescript": "^3.7.5"
+  },
+  "jest": {
+    "errorOnDeprecated": true,
+    "testEnvironment": "node",
+    "testMatch": [
+      "**/?(*.)+(spec|test).js"
+    ]
+  }
+}

packages/@jsii/integ-test/test/build-cdk.test.d.ts

@@ -0,0 +1 @@
+export {};

packages/@jsii/integ-test/test/build-cdk.test.ts

@@ -0,0 +1,81 @@
+// import { IncomingMessage } from 'http';
+import * as fs from 'fs';
+import * as path from 'path';
+import * as Octokit from '@octokit/rest';
+import { downloadReleaseAsset, minutes, ProcessManager } from '../utils';
+import * as dotenv from 'dotenv';
+
+const { mkdtemp } = fs.promises;
+
+dotenv.config();
+const JSII_DIR = path.resolve(require.resolve('jsii'), '..', '..');
+const JSII_PACMAK_DIR = path.resolve(require.resolve('jsii-pacmak'), '..', '..');
+
+const octokit = new Octokit({
+  auth: process.env.GITHUB_TOKEN
+});
+
+describe('Build CDK', () => {
+  let buildDir: string;
+  let processes: ProcessManager;
+
+  beforeAll(async () => {
+    processes = new ProcessManager();
+    buildDir = await mkdtemp(path.join(__dirname, 'build'));
+  });
+
+  afterAll(async () => {
+    await processes.killAll();
+    await rmdirRecursive(buildDir);
+  });
+
+  test('can build latest cdk release', async (done) => {
+    // download latest release info
+    console.time('cdkbuild');
+    const release = await octokit.repos.getLatestRelease({
+      owner: 'aws',
+      repo: 'aws-cdk'
+    });
+
+    // save code to tmp dir
+    const fileName = 'cdk.tar.gz';
+    const tarFile = path.join(buildDir, fileName);
+    const code = await downloadReleaseAsset(`https://api.github.com/repos/aws/aws-cdk/tarball/${release.data.tag_name}`);
+    const codeStream = fs.createWriteStream(tarFile);
+
+    // save to file and wait to finish
+    code.pipe(codeStream);
+    await new Promise(resolve => codeStream.on('close', () => {
+      resolve();
+    }));
+
+    // unzip tar archive
+    await processes.spawn('tar', ['-xzvf', fileName], {
+      cwd: buildDir
+    });
+
+    // root dir of extracted src
+    // `${buildDir}/${owner}-${repo}-${first 7 chars of commit hash}
+    const srcDir = path.join(buildDir, `aws-aws-cdk-${release.data.target_commitish.substring(0, 7)}`);
+
+    // install cdk dependencies
+    await processes.spawn('yarn', ['install'], {
+      cwd: srcDir
+    });
+
+    // link local jsii/jsii-pacmak builds
+    await processes.spawn('rm', ['-rf', './node_modules/jsii'], { cwd: srcDir });
+    await processes.spawn('rm', ['-rf', './node_modules/jsii-pacmak'], { cwd: srcDir });
+    await processes.spawn('ln', ['-s', JSII_DIR, './node_modules'], { cwd: srcDir });
+    await processes.spawn('ln', ['-s', JSII_PACMAK_DIR, './node_modules'], { cwd: srcDir });
+
+    // build cdk
+    await processes.spawn('./node_modules/.bin/lerna', ['run', 'build', '--stream'], { cwd: srcDir });
+
+    // package modules
+    await processes.spawn('yarn', ['run', 'pack'], { cwd: srcDir });
+    console.timeEnd('cdkbuild');
+    done();
+
+  }, minutes(60));
+});

packages/@jsii/integ-test/utils/index.d.ts

@@ -0,0 +1,32 @@
+/// <reference types="node" />
+import * as cp from 'child_process';
+import { PathLike } from 'fs';
+export declare const minutes: (num: number) => number;
+/**
+ * rmdirRecursive
+ *
+ * recursive directory removal for cleanup after build test. Node10 fs module
+ * doesn't support the `recursive` option
+ */
+export declare const rmdirRecursive: (dir: PathLike) => Promise<void>;
+export declare class ProcessManager {
+    processes: {
+        [pid: string]: {
+            proc: cp.ChildProcess;
+            promise: Promise<void>;
+        };
+    };
+    constructor();
+    killAll(): Promise<void>;
+    private add;
+    private remove;
+    spawn(cmd: string, args: string[], opts: any): Promise<void>;
+}
+/**
+ * downloadReleaseAsset
+ *
+ * Wrap http calls to download release asset in a promise. Github responds with
+ * a 302 sometimes which is required to be handled. Returns the buffer to be
+ * streamed to destination fs stream.
+ */
+export declare const downloadReleaseAsset: (url: string) => Promise<any>;

packages/@jsii/integ-test/utils/index.ts

@@ -0,0 +1,118 @@
+import * as cp from 'child_process';
+import * as https from 'https';
+import * as path from 'path';
+import { PathLike, promises as fs } from 'fs';
+
+export const minutes = (num: number) => num * 1000 * 60
+
+/**
+ * rmdirRecursive
+ *
+ * recursive directory removal for cleanup after build test. Node10 fs module
+ * doesn't support the `recursive` option
+ */
+export const rmdirRecursive = async (dir: PathLike) => {
+  const contents = await fs.readdir(dir);
+  await Promise.all(contents.map(async (file) => {
+    const currentPath = path.join(dir.toString(), file);
+    if ((await fs.lstat(currentPath)).isDirectory()) {
+      await rmdirRecursive(currentPath);
+    } else {
+      await fs.unlink(currentPath);
+    }
+  }));
+
+  await fs.rmdir(dir);
+};
+
+/*
+ * ProcessManager
+ *
+ * Used to track and clean up processes if tests fail or timeout
+ */
+export class ProcessManager {
+  processes: {
+    [pid: string]: {
+      proc: cp.ChildProcess,
+      promise: Promise<void>
+    }
+  };
+
+  constructor() {
+    this.processes = {};
+  }
+
+  async killAll() {
+    const values = Object.values(this.processes);
+    values.forEach(procObj => procObj.proc.kill());
+    await Promise.all(values.map(proc => proc.promise));
+    this.processes = {};
+  }
+
+  private add(proc: cp.ChildProcess, promise: Promise<void>) {
+    const { pid } = proc;
+    this.processes[pid] = { proc, promise };
+  }
+
+  private remove(proc: cp.ChildProcess) {
+    delete this.processes[proc.pid];
+  }
+
+  spawn(cmd: string, args: string[], opts: any) {
+    const proc = cp.spawn(cmd, args, opts);
+    proc.stdout.pipe(process.stdout);
+    proc.stderr.pipe(process.stderr);
+
+    const promise: Promise<void> = new Promise((resolve, reject) => {
+      proc.on('exit', code => {
+        const message = `child process exited with code: ${code}`;
+        if (code !== 0) {
+          process.stderr.write(message);
+          reject(new Error(message));
+        } else {
+          process.stdout.write(message);
+          resolve();
+        }
+
+        this.remove(proc);
+      });
+
+      proc.on('error', error => {
+        process.stderr.write(`Process ${proc.pid} error: ${error}`);
+      });
+    });
+
+    this.add(proc, promise);
+    return promise;
+  }
+}
+
+/**
+ * downloadReleaseAsset
+ *
+ * Wrap http calls to download release asset in a promise. Github responds with
+ * a 302 sometimes which is required to be handled. Returns the buffer to be
+ * streamed to destination fs stream.
+ */
+export const downloadReleaseAsset = (url: string): Promise<any> => new Promise((resolve, reject) => {
+  const config = {
+    headers: {
+      'User-Agent': 'aws-cdk',
+      Authorization: `token ${process.env.GITHUB_TOKEN}`,
+      Accept: 'application/json'
+    }
+  };
+
+  https.get(url, config, response => {
+    if (response.statusCode! < 200 && response.statusCode !== 302) {
+      reject(new Error(`Status Code: ${response.statusCode}`));
+    }
+    if (response.statusCode === 302 && response.headers.location) {
+      return https.get(response.headers.location, config, response => {
+        return resolve(response);
+      });
+    }
+
+    return resolve(response);
+  });
+});

yarn.lock

@@ -1230,6 +1230,13 @@
   resolved "https://registry.yarnpkg.com/@nodelib/fs.stat/-/fs.stat-1.1.3.tgz#2b5a3ab3f918cca48a8c754c08168e3f03eba61b"
   integrity sha512-shAmDyaQC4H92APFoIaVDHCx5bStIocgvbwQyxPRrbUY20V1EYTbSDchWbuwlMG3V17cprZhA6+78JfB+3DTPw==
 
+"@octokit/auth-token@^2.4.0":
+  version "2.4.0"
+  resolved "https://registry.yarnpkg.com/@octokit/auth-token/-/auth-token-2.4.0.tgz#b64178975218b99e4dfe948253f0673cbbb59d9f"
+  integrity sha512-eoOVMjILna7FVQf96iWc3+ZtE/ZT6y8ob8ZzcqKY1ibSQCnu4O/B7pJvzMx5cyZ/RjAff6DAdEb0O0Cjcxidkg==
+  dependencies:
+    "@octokit/types" "^2.0.0"
+
 "@octokit/endpoint@^5.5.0":
   version "5.5.1"
   resolved "https://registry.yarnpkg.com/@octokit/endpoint/-/endpoint-5.5.1.tgz#2eea81e110ca754ff2de11c79154ccab4ae16b3f"
@@ -1285,6 +1292,25 @@
     once "^1.4.0"
     universal-user-agent "^4.0.0"
 
+"@octokit/rest@^16.36.0":
+  version "16.38.1"
+  resolved "https://registry.yarnpkg.com/@octokit/rest/-/rest-16.38.1.tgz#be24e0faa7d0bdb9459fbc089ec866ed11774b72"
+  integrity sha512-zyNFx+/Bd1EXt7LQjfrc6H4wryBQ/oDuZeZhGMBSFr1eMPFDmpEweFQR3R25zjKwBQpDY7L5GQO6A3XSaOfV1w==
+  dependencies:
+    "@octokit/auth-token" "^2.4.0"
+    "@octokit/request" "^5.2.0"
+    "@octokit/request-error" "^1.0.2"
+    atob-lite "^2.0.0"
+    before-after-hook "^2.0.0"
+    btoa-lite "^1.0.0"
+    deprecation "^2.0.0"
+    lodash.get "^4.4.2"
+    lodash.set "^4.3.2"
+    lodash.uniq "^4.5.0"
+    octokit-pagination-methods "^1.1.0"
+    once "^1.4.0"
+    universal-user-agent "^4.0.0"
+
 "@octokit/types@^2.0.0":
   version "2.0.2"
   resolved "https://registry.yarnpkg.com/@octokit/types/-/types-2.0.2.tgz#0888497f5a664e28b0449731d5e88e19b2a74f90"
@@ -3231,6 +3257,11 @@ dot-prop@^4.2.0:
   dependencies:
     is-obj "^1.0.0"
 
+dotenv@^8.2.0:
+  version "8.2.0"
+  resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-8.2.0.tgz#97e619259ada750eea3e4ea3e26bceea5424b16a"
+  integrity sha512-8sJ78ElpbDJBHNeBzUbUVLsqKdccaa/BXF1uPTw3GrvQTBgrQrtObr2mUrE38vzYd8cEv+m/JBfDLioYcfXoaw==
+
 duplexer@^0.1.1:
   version "0.1.1"
   resolved "https://registry.yarnpkg.com/duplexer/-/duplexer-0.1.1.tgz#ace6ff808c1ce66b57d1ebf97977acb02334cfc1"
@@ -8499,7 +8530,7 @@ typescript-json-schema@^0.42.0:
     typescript "^3.5.3"
     yargs "^14.0.0"
 
-typescript@^3.5.3, typescript@~3.7.5:
+typescript@^3.5.3, typescript@^3.7.5, typescript@~3.7.5:
   version "3.7.5"
   resolved "https://registry.yarnpkg.com/typescript/-/typescript-3.7.5.tgz#0692e21f65fd4108b9330238aac11dd2e177a1ae"
   integrity sha512-/P5lkRXkWHNAbcJIiHPfRoKqyd7bsyCma1hZNUGfn20qm64T6ZBlrzprymeu918H+mB/0rIg2gGK/BXkhhYgBw==