From 4d83bcd74ce3039e757d6fc26510b47b442657a2 Mon Sep 17 00:00:00 2001 From: Mitchell Valine Date: Tue, 21 Jan 2020 16:56:42 -0800 Subject: [PATCH] feat: CDK Build Integration Test Adds @jsii/integ-test private module for defining new integration tests. Adds a new integration test that downloads the latest CDK release source code and builds it with the local version of jsii and jsii-pacmak. This unit test requires a github access token defined in the environment to get the latest release version and download the asset. Fixes: #1209 --- package.json | 1 + packages/@jsii/integ-test/.env.example | 1 + packages/@jsii/integ-test/.gitignore | 2 + packages/@jsii/integ-test/README.md | 10 ++ packages/@jsii/integ-test/package.json | 29 +++++ .../@jsii/integ-test/test/build-cdk.test.d.ts | 1 + .../@jsii/integ-test/test/build-cdk.test.ts | 81 ++++++++++++ packages/@jsii/integ-test/utils/index.d.ts | 32 +++++ packages/@jsii/integ-test/utils/index.ts | 118 ++++++++++++++++++ yarn.lock | 33 ++++- 10 files changed, 307 insertions(+), 1 deletion(-) create mode 100644 packages/@jsii/integ-test/.env.example create mode 100644 packages/@jsii/integ-test/.gitignore create mode 100644 packages/@jsii/integ-test/README.md create mode 100644 packages/@jsii/integ-test/package.json create mode 100644 packages/@jsii/integ-test/test/build-cdk.test.d.ts create mode 100644 packages/@jsii/integ-test/test/build-cdk.test.ts create mode 100644 packages/@jsii/integ-test/utils/index.d.ts create mode 100644 packages/@jsii/integ-test/utils/index.ts diff --git a/package.json b/package.json index 6074be04ae..84b9105989 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,7 @@ "fetch-dotnet-snk": "bash scripts/fetch-dotnet-snk.sh", "package": "bash scripts/package.sh", "test": "lerna run test --stream", + "test:integ": "lerna run test:integ --stream", "test:update": "lerna run test:update --stream" }, "devDependencies": { diff --git a/packages/@jsii/integ-test/.env.example b/packages/@jsii/integ-test/.env.example new file mode 100644 index 0000000000..46f8a2c396 --- /dev/null +++ b/packages/@jsii/integ-test/.env.example @@ -0,0 +1 @@ +GITHUB_TOKEN=personal_access_token diff --git a/packages/@jsii/integ-test/.gitignore b/packages/@jsii/integ-test/.gitignore new file mode 100644 index 0000000000..4ec96f4e9d --- /dev/null +++ b/packages/@jsii/integ-test/.gitignore @@ -0,0 +1,2 @@ +.env +*.js diff --git a/packages/@jsii/integ-test/README.md b/packages/@jsii/integ-test/README.md new file mode 100644 index 0000000000..a6a8bb9bec --- /dev/null +++ b/packages/@jsii/integ-test/README.md @@ -0,0 +1,10 @@ +# JSII Integration Tests + +A suite of integration tests for JSII and related modules. + +## Running + +Running the integration tests locally requires a github access token. Copy the +.env.example file and replace the dummy value with a personal access token. + +then run `yarn run test:integ` diff --git a/packages/@jsii/integ-test/package.json b/packages/@jsii/integ-test/package.json new file mode 100644 index 0000000000..bc477b5115 --- /dev/null +++ b/packages/@jsii/integ-test/package.json @@ -0,0 +1,29 @@ +{ + "name": "integ-test", + "version": "1.0.0", + "description": "", + "main": "index.js", + "private": true, + "scripts": { + "build": "tsc", + "test:integ": "jest" + }, + "keywords": [], + "author": "", + "license": "ISC", + "dependencies": { + "@octokit/rest": "^16.36.0", + "dotenv": "^8.2.0", + "jest": "^25.1.0", + "jsii": "^0.21.2", + "jsii-pacmak": "^0.21.2", + "typescript": "^3.7.5" + }, + "jest": { + "errorOnDeprecated": true, + "testEnvironment": "node", + "testMatch": [ + "**/?(*.)+(spec|test).js" + ] + } +} diff --git a/packages/@jsii/integ-test/test/build-cdk.test.d.ts b/packages/@jsii/integ-test/test/build-cdk.test.d.ts new file mode 100644 index 0000000000..cb0ff5c3b5 --- /dev/null +++ b/packages/@jsii/integ-test/test/build-cdk.test.d.ts @@ -0,0 +1 @@ +export {}; diff --git a/packages/@jsii/integ-test/test/build-cdk.test.ts b/packages/@jsii/integ-test/test/build-cdk.test.ts new file mode 100644 index 0000000000..317c366115 --- /dev/null +++ b/packages/@jsii/integ-test/test/build-cdk.test.ts @@ -0,0 +1,81 @@ +// import { IncomingMessage } from 'http'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as Octokit from '@octokit/rest'; +import { downloadReleaseAsset, minutes, ProcessManager } from '../utils'; +import * as dotenv from 'dotenv'; + +const { mkdtemp } = fs.promises; + +dotenv.config(); +const JSII_DIR = path.resolve(require.resolve('jsii'), '..', '..'); +const JSII_PACMAK_DIR = path.resolve(require.resolve('jsii-pacmak'), '..', '..'); + +const octokit = new Octokit({ + auth: process.env.GITHUB_TOKEN +}); + +describe('Build CDK', () => { + let buildDir: string; + let processes: ProcessManager; + + beforeAll(async () => { + processes = new ProcessManager(); + buildDir = await mkdtemp(path.join(__dirname, 'build')); + }); + + afterAll(async () => { + await processes.killAll(); + await rmdirRecursive(buildDir); + }); + + test('can build latest cdk release', async (done) => { + // download latest release info + console.time('cdkbuild'); + const release = await octokit.repos.getLatestRelease({ + owner: 'aws', + repo: 'aws-cdk' + }); + + // save code to tmp dir + const fileName = 'cdk.tar.gz'; + const tarFile = path.join(buildDir, fileName); + const code = await downloadReleaseAsset(`https://api.github.com/repos/aws/aws-cdk/tarball/${release.data.tag_name}`); + const codeStream = fs.createWriteStream(tarFile); + + // save to file and wait to finish + code.pipe(codeStream); + await new Promise(resolve => codeStream.on('close', () => { + resolve(); + })); + + // unzip tar archive + await processes.spawn('tar', ['-xzvf', fileName], { + cwd: buildDir + }); + + // root dir of extracted src + // `${buildDir}/${owner}-${repo}-${first 7 chars of commit hash} + const srcDir = path.join(buildDir, `aws-aws-cdk-${release.data.target_commitish.substring(0, 7)}`); + + // install cdk dependencies + await processes.spawn('yarn', ['install'], { + cwd: srcDir + }); + + // link local jsii/jsii-pacmak builds + await processes.spawn('rm', ['-rf', './node_modules/jsii'], { cwd: srcDir }); + await processes.spawn('rm', ['-rf', './node_modules/jsii-pacmak'], { cwd: srcDir }); + await processes.spawn('ln', ['-s', JSII_DIR, './node_modules'], { cwd: srcDir }); + await processes.spawn('ln', ['-s', JSII_PACMAK_DIR, './node_modules'], { cwd: srcDir }); + + // build cdk + await processes.spawn('./node_modules/.bin/lerna', ['run', 'build', '--stream'], { cwd: srcDir }); + + // package modules + await processes.spawn('yarn', ['run', 'pack'], { cwd: srcDir }); + console.timeEnd('cdkbuild'); + done(); + + }, minutes(60)); +}); diff --git a/packages/@jsii/integ-test/utils/index.d.ts b/packages/@jsii/integ-test/utils/index.d.ts new file mode 100644 index 0000000000..0d7366fd3e --- /dev/null +++ b/packages/@jsii/integ-test/utils/index.d.ts @@ -0,0 +1,32 @@ +/// +import * as cp from 'child_process'; +import { PathLike } from 'fs'; +export declare const minutes: (num: number) => number; +/** + * rmdirRecursive + * + * recursive directory removal for cleanup after build test. Node10 fs module + * doesn't support the `recursive` option + */ +export declare const rmdirRecursive: (dir: PathLike) => Promise; +export declare class ProcessManager { + processes: { + [pid: string]: { + proc: cp.ChildProcess; + promise: Promise; + }; + }; + constructor(); + killAll(): Promise; + private add; + private remove; + spawn(cmd: string, args: string[], opts: any): Promise; +} +/** + * downloadReleaseAsset + * + * Wrap http calls to download release asset in a promise. Github responds with + * a 302 sometimes which is required to be handled. Returns the buffer to be + * streamed to destination fs stream. + */ +export declare const downloadReleaseAsset: (url: string) => Promise; diff --git a/packages/@jsii/integ-test/utils/index.ts b/packages/@jsii/integ-test/utils/index.ts new file mode 100644 index 0000000000..d90f50ebb3 --- /dev/null +++ b/packages/@jsii/integ-test/utils/index.ts @@ -0,0 +1,118 @@ +import * as cp from 'child_process'; +import * as https from 'https'; +import * as path from 'path'; +import { PathLike, promises as fs } from 'fs'; + +export const minutes = (num: number) => num * 1000 * 60 + +/** + * rmdirRecursive + * + * recursive directory removal for cleanup after build test. Node10 fs module + * doesn't support the `recursive` option + */ +export const rmdirRecursive = async (dir: PathLike) => { + const contents = await fs.readdir(dir); + await Promise.all(contents.map(async (file) => { + const currentPath = path.join(dir.toString(), file); + if ((await fs.lstat(currentPath)).isDirectory()) { + await rmdirRecursive(currentPath); + } else { + await fs.unlink(currentPath); + } + })); + + await fs.rmdir(dir); +}; + +/* + * ProcessManager + * + * Used to track and clean up processes if tests fail or timeout + */ +export class ProcessManager { + processes: { + [pid: string]: { + proc: cp.ChildProcess, + promise: Promise + } + }; + + constructor() { + this.processes = {}; + } + + async killAll() { + const values = Object.values(this.processes); + values.forEach(procObj => procObj.proc.kill()); + await Promise.all(values.map(proc => proc.promise)); + this.processes = {}; + } + + private add(proc: cp.ChildProcess, promise: Promise) { + const { pid } = proc; + this.processes[pid] = { proc, promise }; + } + + private remove(proc: cp.ChildProcess) { + delete this.processes[proc.pid]; + } + + spawn(cmd: string, args: string[], opts: any) { + const proc = cp.spawn(cmd, args, opts); + proc.stdout.pipe(process.stdout); + proc.stderr.pipe(process.stderr); + + const promise: Promise = new Promise((resolve, reject) => { + proc.on('exit', code => { + const message = `child process exited with code: ${code}`; + if (code !== 0) { + process.stderr.write(message); + reject(new Error(message)); + } else { + process.stdout.write(message); + resolve(); + } + + this.remove(proc); + }); + + proc.on('error', error => { + process.stderr.write(`Process ${proc.pid} error: ${error}`); + }); + }); + + this.add(proc, promise); + return promise; + } +} + +/** + * downloadReleaseAsset + * + * Wrap http calls to download release asset in a promise. Github responds with + * a 302 sometimes which is required to be handled. Returns the buffer to be + * streamed to destination fs stream. + */ +export const downloadReleaseAsset = (url: string): Promise => new Promise((resolve, reject) => { + const config = { + headers: { + 'User-Agent': 'aws-cdk', + Authorization: `token ${process.env.GITHUB_TOKEN}`, + Accept: 'application/json' + } + }; + + https.get(url, config, response => { + if (response.statusCode! < 200 && response.statusCode !== 302) { + reject(new Error(`Status Code: ${response.statusCode}`)); + } + if (response.statusCode === 302 && response.headers.location) { + return https.get(response.headers.location, config, response => { + return resolve(response); + }); + } + + return resolve(response); + }); +}); diff --git a/yarn.lock b/yarn.lock index 611e567ae1..e7f8bd174f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1230,6 +1230,13 @@ resolved "https://registry.yarnpkg.com/@nodelib/fs.stat/-/fs.stat-1.1.3.tgz#2b5a3ab3f918cca48a8c754c08168e3f03eba61b" integrity sha512-shAmDyaQC4H92APFoIaVDHCx5bStIocgvbwQyxPRrbUY20V1EYTbSDchWbuwlMG3V17cprZhA6+78JfB+3DTPw== +"@octokit/auth-token@^2.4.0": + version "2.4.0" + resolved "https://registry.yarnpkg.com/@octokit/auth-token/-/auth-token-2.4.0.tgz#b64178975218b99e4dfe948253f0673cbbb59d9f" + integrity sha512-eoOVMjILna7FVQf96iWc3+ZtE/ZT6y8ob8ZzcqKY1ibSQCnu4O/B7pJvzMx5cyZ/RjAff6DAdEb0O0Cjcxidkg== + dependencies: + "@octokit/types" "^2.0.0" + "@octokit/endpoint@^5.5.0": version "5.5.1" resolved "https://registry.yarnpkg.com/@octokit/endpoint/-/endpoint-5.5.1.tgz#2eea81e110ca754ff2de11c79154ccab4ae16b3f" @@ -1285,6 +1292,25 @@ once "^1.4.0" universal-user-agent "^4.0.0" +"@octokit/rest@^16.36.0": + version "16.38.1" + resolved "https://registry.yarnpkg.com/@octokit/rest/-/rest-16.38.1.tgz#be24e0faa7d0bdb9459fbc089ec866ed11774b72" + integrity sha512-zyNFx+/Bd1EXt7LQjfrc6H4wryBQ/oDuZeZhGMBSFr1eMPFDmpEweFQR3R25zjKwBQpDY7L5GQO6A3XSaOfV1w== + dependencies: + "@octokit/auth-token" "^2.4.0" + "@octokit/request" "^5.2.0" + "@octokit/request-error" "^1.0.2" + atob-lite "^2.0.0" + before-after-hook "^2.0.0" + btoa-lite "^1.0.0" + deprecation "^2.0.0" + lodash.get "^4.4.2" + lodash.set "^4.3.2" + lodash.uniq "^4.5.0" + octokit-pagination-methods "^1.1.0" + once "^1.4.0" + universal-user-agent "^4.0.0" + "@octokit/types@^2.0.0": version "2.0.2" resolved "https://registry.yarnpkg.com/@octokit/types/-/types-2.0.2.tgz#0888497f5a664e28b0449731d5e88e19b2a74f90" @@ -3231,6 +3257,11 @@ dot-prop@^4.2.0: dependencies: is-obj "^1.0.0" +dotenv@^8.2.0: + version "8.2.0" + resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-8.2.0.tgz#97e619259ada750eea3e4ea3e26bceea5424b16a" + integrity sha512-8sJ78ElpbDJBHNeBzUbUVLsqKdccaa/BXF1uPTw3GrvQTBgrQrtObr2mUrE38vzYd8cEv+m/JBfDLioYcfXoaw== + duplexer@^0.1.1: version "0.1.1" resolved "https://registry.yarnpkg.com/duplexer/-/duplexer-0.1.1.tgz#ace6ff808c1ce66b57d1ebf97977acb02334cfc1" @@ -8499,7 +8530,7 @@ typescript-json-schema@^0.42.0: typescript "^3.5.3" yargs "^14.0.0" -typescript@^3.5.3, typescript@~3.7.5: +typescript@^3.5.3, typescript@^3.7.5, typescript@~3.7.5: version "3.7.5" resolved "https://registry.yarnpkg.com/typescript/-/typescript-3.7.5.tgz#0692e21f65fd4108b9330238aac11dd2e177a1ae" integrity sha512-/P5lkRXkWHNAbcJIiHPfRoKqyd7bsyCma1hZNUGfn20qm64T6ZBlrzprymeu918H+mB/0rIg2gGK/BXkhhYgBw==