diff --git a/packages/@aws-cdk/assets-docker/lib/image-asset.ts b/packages/@aws-cdk/assets-docker/lib/image-asset.ts index a968d8794f88f..7d939dfca904d 100644 --- a/packages/@aws-cdk/assets-docker/lib/image-asset.ts +++ b/packages/@aws-cdk/assets-docker/lib/image-asset.ts @@ -1,3 +1,4 @@ +import assets = require('@aws-cdk/assets'); import ecr = require('@aws-cdk/aws-ecr'); import cdk = require('@aws-cdk/cdk'); import cxapi = require('@aws-cdk/cx-api'); @@ -49,14 +50,20 @@ export class DockerImageAsset extends cdk.Construct { super(scope, id); // resolve full path - this.directory = path.resolve(props.directory); - if (!fs.existsSync(this.directory)) { - throw new Error(`Cannot find image directory at ${this.directory}`); + const dir = path.resolve(props.directory); + if (!fs.existsSync(dir)) { + throw new Error(`Cannot find image directory at ${dir}`); } - if (!fs.existsSync(path.join(this.directory, 'Dockerfile'))) { - throw new Error(`No 'Dockerfile' found in ${this.directory}`); + if (!fs.existsSync(path.join(dir, 'Dockerfile'))) { + throw new Error(`No 'Dockerfile' found in ${dir}`); } + const staging = new assets.Staging(this, 'Staging', { + sourcePath: dir + }); + + this.directory = staging.stagedPath; + const imageNameParameter = new cdk.CfnParameter(this, 'ImageName', { type: 'String', description: `ECR repository name and tag asset "${this.node.path}"`, diff --git a/packages/@aws-cdk/assets-docker/package-lock.json b/packages/@aws-cdk/assets-docker/package-lock.json index f64c37465ff3c..137155b2c29d2 100644 --- a/packages/@aws-cdk/assets-docker/package-lock.json +++ b/packages/@aws-cdk/assets-docker/package-lock.json @@ -1,6 +1,6 @@ { "name": "@aws-cdk/assets-docker", - "version": "0.26.0", + "version": "0.28.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/packages/@aws-cdk/assets-docker/package.json b/packages/@aws-cdk/assets-docker/package.json index ae62d67d5bcba..bd0c9111e8c0a 100644 --- a/packages/@aws-cdk/assets-docker/package.json +++ b/packages/@aws-cdk/assets-docker/package.json @@ -69,6 +69,7 @@ "@aws-cdk/aws-iam": "^0.28.0", "@aws-cdk/aws-lambda": "^0.28.0", "@aws-cdk/aws-s3": "^0.28.0", + "@aws-cdk/assets": "^0.28.0", "@aws-cdk/cdk": "^0.28.0", "@aws-cdk/cx-api": "^0.28.0" }, @@ -76,6 +77,7 @@ "peerDependencies": { "@aws-cdk/aws-ecr": "^0.28.0", "@aws-cdk/aws-iam": "^0.28.0", + "@aws-cdk/assets": "^0.28.0", "@aws-cdk/aws-s3": "^0.28.0", "@aws-cdk/cdk": "^0.28.0" }, diff --git a/packages/@aws-cdk/assets-docker/test/test.image-asset.ts b/packages/@aws-cdk/assets-docker/test/test.image-asset.ts index ed0b5ae2b51c9..43cc5c5705989 100644 --- a/packages/@aws-cdk/assets-docker/test/test.image-asset.ts +++ b/packages/@aws-cdk/assets-docker/test/test.image-asset.ts @@ -1,7 +1,10 @@ import { expect, haveResource, SynthUtils } from '@aws-cdk/assert'; import iam = require('@aws-cdk/aws-iam'); import cdk = require('@aws-cdk/cdk'); +import cxapi = require('@aws-cdk/cx-api'); +import fs = require('fs'); import { Test } from 'nodeunit'; +import os = require('os'); import path = require('path'); import { DockerImageAsset } from '../lib'; @@ -143,5 +146,30 @@ export = { }); }, /No 'Dockerfile' found in/); test.done(); + }, + + 'docker directory is staged if asset staging is enabled'(test: Test) { + const workdir = mkdtempSync(); + process.chdir(workdir); + + const app = new cdk.App({ + context: { [cxapi.ASSET_STAGING_DIR_CONTEXT]: '.stage-me' } + }); + + const stack = new cdk.Stack(app, 'stack'); + + new DockerImageAsset(stack, 'MyAsset', { + directory: path.join(__dirname, 'demo-image') + }); + + app.run(); + + test.ok(fs.existsSync('.stage-me/96e3ffe92a19cbaa6c558942f7a60246/Dockerfile')); + test.ok(fs.existsSync('.stage-me/96e3ffe92a19cbaa6c558942f7a60246/index.py')); + test.done(); } }; + +function mkdtempSync() { + return fs.mkdtempSync(path.join(os.tmpdir(), 'test.assets')); +} diff --git a/packages/@aws-cdk/assets/lib/asset.ts b/packages/@aws-cdk/assets/lib/asset.ts index d985af72954b7..e601fd73e2625 100644 --- a/packages/@aws-cdk/assets/lib/asset.ts +++ b/packages/@aws-cdk/assets/lib/asset.ts @@ -4,6 +4,7 @@ import cdk = require('@aws-cdk/cdk'); import cxapi = require('@aws-cdk/cx-api'); import fs = require('fs'); import path = require('path'); +import { Staging } from './staging'; /** * Defines the way an asset is packaged before it is uploaded to S3. @@ -61,7 +62,10 @@ export class Asset extends cdk.Construct { public readonly s3Url: string; /** - * Resolved full-path location of this asset. + * The path to the asset (stringinfied token). + * + * If asset staging is disabled, this will just be the original path. + * If asset staging is enabled it will be the staged path. */ public readonly assetPath: string; @@ -84,16 +88,20 @@ export class Asset extends cdk.Construct { constructor(scope: cdk.Construct, id: string, props: GenericAssetProps) { super(scope, id); - // resolve full path - this.assetPath = path.resolve(props.path); + // stage the asset source (conditionally). + const staging = new Staging(this, 'Stage', { + sourcePath: path.resolve(props.path) + }); + + this.assetPath = staging.stagedPath; // sets isZipArchive based on the type of packaging and file extension const allowedExtensions: string[] = ['.jar', '.zip']; this.isZipArchive = props.packaging === AssetPackaging.ZipDirectory ? true - : allowedExtensions.some(ext => this.assetPath.toLowerCase().endsWith(ext)); + : allowedExtensions.some(ext => staging.sourcePath.toLowerCase().endsWith(ext)); - validateAssetOnDisk(this.assetPath, props.packaging); + validateAssetOnDisk(staging.sourcePath, props.packaging); // add parameters for s3 bucket and s3 key. those will be set by // the toolkit or by CI/CD when the stack is deployed and will include diff --git a/packages/@aws-cdk/assets/lib/fs/copy.ts b/packages/@aws-cdk/assets/lib/fs/copy.ts new file mode 100644 index 0000000000000..6ea1f2a6e5f8c --- /dev/null +++ b/packages/@aws-cdk/assets/lib/fs/copy.ts @@ -0,0 +1,89 @@ +import fs = require('fs'); +import minimatch = require('minimatch'); +import path = require('path'); +import { FollowMode } from './follow-mode'; + +export interface CopyOptions { + /** + * @default External only follows symlinks that are external to the source directory + */ + follow?: FollowMode; + + /** + * glob patterns to exclude from the copy. + */ + exclude?: string[]; +} + +export function copyDirectory(srcDir: string, destDir: string, options: CopyOptions = { }, rootDir?: string) { + const follow = options.follow !== undefined ? options.follow : FollowMode.External; + const exclude = options.exclude || []; + + rootDir = rootDir || srcDir; + + if (!fs.statSync(srcDir).isDirectory()) { + throw new Error(`${srcDir} is not a directory`); + } + + const files = fs.readdirSync(srcDir); + for (const file of files) { + const sourceFilePath = path.join(srcDir, file); + + if (shouldExclude(path.relative(rootDir, sourceFilePath))) { + continue; + } + + const destFilePath = path.join(destDir, file); + + let stat: fs.Stats | undefined = follow === FollowMode.Always + ? fs.statSync(sourceFilePath) + : fs.lstatSync(sourceFilePath); + + if (stat && stat.isSymbolicLink()) { + const target = fs.readlinkSync(sourceFilePath); + + // determine if this is an external link (i.e. the target's absolute path + // is outside of the root directory). + const targetPath = path.normalize(path.resolve(srcDir, target)); + const rootPath = path.normalize(rootDir); + const external = !targetPath.startsWith(rootPath); + + if (follow === FollowMode.External && external) { + stat = fs.statSync(sourceFilePath); + } else { + fs.symlinkSync(target, destFilePath); + stat = undefined; + } + } + + if (stat && stat.isDirectory()) { + fs.mkdirSync(destFilePath); + copyDirectory(sourceFilePath, destFilePath, options, rootDir); + stat = undefined; + } + + if (stat && stat.isFile()) { + fs.copyFileSync(sourceFilePath, destFilePath); + stat = undefined; + } + } + + function shouldExclude(filePath: string): boolean { + let excludeOutput = false; + + for (const pattern of exclude) { + const negate = pattern.startsWith('!'); + const match = minimatch(filePath, pattern, { matchBase: true, flipNegate: true }); + + if (!negate && match) { + excludeOutput = true; + } + + if (negate && match) { + excludeOutput = false; + } + } + + return excludeOutput; + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/assets/lib/fs/fingerprint.ts b/packages/@aws-cdk/assets/lib/fs/fingerprint.ts new file mode 100644 index 0000000000000..06cdb6a0ed2aa --- /dev/null +++ b/packages/@aws-cdk/assets/lib/fs/fingerprint.ts @@ -0,0 +1,86 @@ +import crypto = require('crypto'); +import fs = require('fs'); +import path = require('path'); +import { FollowMode } from './follow-mode'; + +const BUFFER_SIZE = 8 * 1024; + +export interface FingerprintOptions { + /** + * Extra information to encode into the fingerprint (e.g. build instructions + * and other inputs) + */ + extra?: string; + + /** + * List of exclude patterns (see `CopyOptions`) + * @default include all files + */ + exclude?: string[]; + + /** + * What to do when we encounter symlinks. + * @default External only follows symlinks that are external to the source + * directory + */ + follow?: FollowMode; +} + +/** + * Produces fingerprint based on the contents of a single file or an entire directory tree. + * + * The fingerprint will also include: + * 1. An extra string if defined in `options.extra`. + * 2. The set of exclude patterns, if defined in `options.exclude` + * 3. The symlink follow mode value. + * + * @param fileOrDirectory The directory or file to fingerprint + * @param options Fingerprinting options + */ +export function fingerprint(fileOrDirectory: string, options: FingerprintOptions = { }) { + const follow = options.follow !== undefined ? options.follow : FollowMode.External; + const hash = crypto.createHash('md5'); + addToHash(fileOrDirectory); + + hash.update(`==follow==${follow}==\n\n`); + + if (options.extra) { + hash.update(`==extra==${options.extra}==\n\n`); + } + + for (const ex of options.exclude || []) { + hash.update(`==exclude==${ex}==\n\n`); + } + + return hash.digest('hex'); + + function addToHash(pathToAdd: string) { + hash.update('==\n'); + const relativePath = path.relative(fileOrDirectory, pathToAdd); + hash.update(relativePath + '\n'); + hash.update('~~~~~~~~~~~~~~~~~~\n'); + const stat = fs.statSync(pathToAdd); + + if (stat.isSymbolicLink()) { + const target = fs.readlinkSync(pathToAdd); + hash.update(target); + } else if (stat.isDirectory()) { + for (const file of fs.readdirSync(pathToAdd)) { + addToHash(path.join(pathToAdd, file)); + } + } else { + const file = fs.openSync(pathToAdd, 'r'); + const buffer = Buffer.alloc(BUFFER_SIZE); + + try { + let bytesRead; + do { + bytesRead = fs.readSync(file, buffer, 0, BUFFER_SIZE, null); + hash.update(buffer.slice(0, bytesRead)); + } while (bytesRead === BUFFER_SIZE); + } finally { + fs.closeSync(file); + } + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/assets/lib/fs/follow-mode.ts b/packages/@aws-cdk/assets/lib/fs/follow-mode.ts new file mode 100644 index 0000000000000..02ecebfaaa0a7 --- /dev/null +++ b/packages/@aws-cdk/assets/lib/fs/follow-mode.ts @@ -0,0 +1,29 @@ +export enum FollowMode { + /** + * Never follow symlinks. + */ + Never = 'never', + + /** + * Materialize all symlinks, whether they are internal or external to the source directory. + */ + Always = 'always', + + /** + * Only follows symlinks that are external to the source directory. + */ + External = 'external', + + // ----------------- TODO:::::::::::::::::::::::::::::::::::::::::::: + /** + * Forbids source from having any symlinks pointing outside of the source + * tree. + * + * This is the safest mode of operation as it ensures that copy operations + * won't materialize files from the user's file system. Internal symlinks are + * not followed. + * + * If the copy operation runs into an external symlink, it will fail. + */ + BlockExternal = 'internal-only', +} \ No newline at end of file diff --git a/packages/@aws-cdk/assets/lib/fs/index.ts b/packages/@aws-cdk/assets/lib/fs/index.ts new file mode 100644 index 0000000000000..31b1f468bbdfc --- /dev/null +++ b/packages/@aws-cdk/assets/lib/fs/index.ts @@ -0,0 +1,3 @@ +export * from './fingerprint'; +export * from './follow-mode'; +export * from './copy'; \ No newline at end of file diff --git a/packages/@aws-cdk/assets/lib/index.ts b/packages/@aws-cdk/assets/lib/index.ts index ea2719dd83bd3..24ddffa892f0e 100644 --- a/packages/@aws-cdk/assets/lib/index.ts +++ b/packages/@aws-cdk/assets/lib/index.ts @@ -1 +1,2 @@ export * from './asset'; +export * from './staging'; diff --git a/packages/@aws-cdk/assets/lib/staging.ts b/packages/@aws-cdk/assets/lib/staging.ts new file mode 100644 index 0000000000000..5315d602be6a0 --- /dev/null +++ b/packages/@aws-cdk/assets/lib/staging.ts @@ -0,0 +1,91 @@ +import { Construct, Token } from '@aws-cdk/cdk'; +import cxapi = require('@aws-cdk/cx-api'); +import fs = require('fs'); +import path = require('path'); +import { copyDirectory, fingerprint } from './fs'; + +export interface StageProps { + readonly sourcePath: string; +} + +/** + * Stages a file or directory from a location on the file system into a staging + * directory. + * + * This is controlled by the context key 'aws:cdk:asset-staging-dir' and enabled + * by the CLI by default in order to ensure that when the CDK app exists, all + * assets are available for deployment. Otherwise, if an app references assets + * in temporary locations, those will not be available when it exists (see + * https://github.com/awslabs/aws-cdk/issues/1716). + * + * The `stagedPath` property is a stringified token that represents the location + * of the file or directory after staging. It will be resolved only during the + * "prepare" stage and may be either the original path or the staged path + * depending on the context setting. + * + * The file/directory are staged based on their content hash (fingerprint). This + * means that only if content was changed, copy will happen. + */ +export class Staging extends Construct { + + /** + * The path to the asset (stringinfied token). + * + * If asset staging is disabled, this will just be the original path. + * If asset staging is enabled it will be the staged path. + */ + public readonly stagedPath: string; + + /** + * The path of the asset as it was referenced by the user. + */ + public readonly sourcePath: string; + + /** + * The asset path after "prepare" is called. + * + * If staging is disabled, this will just be the original path. + * If staging is enabled it will be the staged path. + */ + private _preparedAssetPath?: string; + + constructor(scope: Construct, id: string, props: StageProps) { + super(scope, id); + + this.sourcePath = props.sourcePath; + this.stagedPath = new Token(() => this._preparedAssetPath).toString(); + } + + protected prepare() { + const stagingDir = this.node.getContext(cxapi.ASSET_STAGING_DIR_CONTEXT); + if (!stagingDir) { + this._preparedAssetPath = this.sourcePath; + return; + } + + if (!fs.existsSync(stagingDir)) { + fs.mkdirSync(stagingDir); + } + + const hash = fingerprint(this.sourcePath); + const targetPath = path.join(stagingDir, hash + path.extname(this.sourcePath)); + + this._preparedAssetPath = targetPath; + + // asset already staged + if (fs.existsSync(targetPath)) { + return; + } + + // copy file/directory to staging directory + const stat = fs.statSync(this.sourcePath); + if (stat.isFile()) { + fs.copyFileSync(this.sourcePath, targetPath); + } else if (stat.isDirectory()) { + fs.mkdirSync(targetPath); + copyDirectory(this.sourcePath, targetPath); + } else { + throw new Error(`Unknown file type: ${this.sourcePath}`); + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/assets/package-lock.json b/packages/@aws-cdk/assets/package-lock.json new file mode 100644 index 0000000000000..d3b41e065d7a8 --- /dev/null +++ b/packages/@aws-cdk/assets/package-lock.json @@ -0,0 +1,41 @@ +{ + "name": "@aws-cdk/assets", + "version": "0.28.0", + "lockfileVersion": 1, + "requires": true, + "dependencies": { + "@types/minimatch": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.3.tgz", + "integrity": "sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA==", + "dev": true + }, + "balanced-match": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", + "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" + }, + "brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "requires": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" + }, + "minimatch": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", + "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "requires": { + "brace-expansion": "^1.1.7" + } + } + } +} diff --git a/packages/@aws-cdk/assets/package.json b/packages/@aws-cdk/assets/package.json index d2d012b51d7a9..3bc0490fa5f36 100644 --- a/packages/@aws-cdk/assets/package.json +++ b/packages/@aws-cdk/assets/package.json @@ -55,6 +55,7 @@ "license": "Apache-2.0", "devDependencies": { "@aws-cdk/assert": "^0.28.0", + "@types/minimatch": "^3.0.3", "aws-cdk": "^0.28.0", "cdk-build-tools": "^0.28.0", "cdk-integ-tools": "^0.28.0", @@ -64,7 +65,8 @@ "@aws-cdk/aws-iam": "^0.28.0", "@aws-cdk/aws-s3": "^0.28.0", "@aws-cdk/cdk": "^0.28.0", - "@aws-cdk/cx-api": "^0.28.0" + "@aws-cdk/cx-api": "^0.28.0", + "minimatch": "^3.0.4" }, "homepage": "https://github.com/awslabs/aws-cdk", "peerDependencies": { @@ -74,5 +76,8 @@ }, "engines": { "node": ">= 8.10.0" - } -} \ No newline at end of file + }, + "bundledDependencies": [ + "minimatch" + ] +} diff --git a/packages/@aws-cdk/assets/test/fs/fixtures/symlinks/external-dir-link b/packages/@aws-cdk/assets/test/fs/fixtures/symlinks/external-dir-link new file mode 120000 index 0000000000000..b9447033a4279 --- /dev/null +++ b/packages/@aws-cdk/assets/test/fs/fixtures/symlinks/external-dir-link @@ -0,0 +1 @@ +../test1/subdir \ No newline at end of file diff --git a/packages/@aws-cdk/assets/test/fs/fixtures/symlinks/external-link.txt b/packages/@aws-cdk/assets/test/fs/fixtures/symlinks/external-link.txt new file mode 120000 index 0000000000000..267020c936652 --- /dev/null +++ b/packages/@aws-cdk/assets/test/fs/fixtures/symlinks/external-link.txt @@ -0,0 +1 @@ +../test1/subdir2/subdir3/file3.txt \ No newline at end of file diff --git a/packages/@aws-cdk/assets/test/fs/fixtures/symlinks/indirect-external-link.txt b/packages/@aws-cdk/assets/test/fs/fixtures/symlinks/indirect-external-link.txt new file mode 120000 index 0000000000000..907a2a65b1515 --- /dev/null +++ b/packages/@aws-cdk/assets/test/fs/fixtures/symlinks/indirect-external-link.txt @@ -0,0 +1 @@ +external-link.txt \ No newline at end of file diff --git a/packages/@aws-cdk/assets/test/fs/fixtures/symlinks/local-dir-link b/packages/@aws-cdk/assets/test/fs/fixtures/symlinks/local-dir-link new file mode 120000 index 0000000000000..42101049ac31a --- /dev/null +++ b/packages/@aws-cdk/assets/test/fs/fixtures/symlinks/local-dir-link @@ -0,0 +1 @@ +normal-dir \ No newline at end of file diff --git a/packages/@aws-cdk/assets/test/fs/fixtures/symlinks/local-link.txt b/packages/@aws-cdk/assets/test/fs/fixtures/symlinks/local-link.txt new file mode 120000 index 0000000000000..b3c6fdbd8bfad --- /dev/null +++ b/packages/@aws-cdk/assets/test/fs/fixtures/symlinks/local-link.txt @@ -0,0 +1 @@ +normal-file.txt \ No newline at end of file diff --git a/packages/@aws-cdk/assets/test/fs/fixtures/symlinks/normal-dir/file-in-subdir.txt b/packages/@aws-cdk/assets/test/fs/fixtures/symlinks/normal-dir/file-in-subdir.txt new file mode 100644 index 0000000000000..f52de026412b3 --- /dev/null +++ b/packages/@aws-cdk/assets/test/fs/fixtures/symlinks/normal-dir/file-in-subdir.txt @@ -0,0 +1 @@ +file in subdir diff --git a/packages/@aws-cdk/assets/test/fs/fixtures/symlinks/normal-file.txt b/packages/@aws-cdk/assets/test/fs/fixtures/symlinks/normal-file.txt new file mode 100644 index 0000000000000..d627587e36e77 --- /dev/null +++ b/packages/@aws-cdk/assets/test/fs/fixtures/symlinks/normal-file.txt @@ -0,0 +1 @@ +this is a normal file diff --git a/packages/@aws-cdk/assets/test/fs/fixtures/test1/external-link.txt b/packages/@aws-cdk/assets/test/fs/fixtures/test1/external-link.txt new file mode 120000 index 0000000000000..c7ba61290b25a --- /dev/null +++ b/packages/@aws-cdk/assets/test/fs/fixtures/test1/external-link.txt @@ -0,0 +1 @@ +../symlinks/normal-file.txt \ No newline at end of file diff --git a/packages/@aws-cdk/assets/test/fs/fixtures/test1/file1.txt b/packages/@aws-cdk/assets/test/fs/fixtures/test1/file1.txt new file mode 100644 index 0000000000000..e2129701f1a4d --- /dev/null +++ b/packages/@aws-cdk/assets/test/fs/fixtures/test1/file1.txt @@ -0,0 +1 @@ +file1 diff --git a/packages/@aws-cdk/assets/test/fs/fixtures/test1/local-link.txt b/packages/@aws-cdk/assets/test/fs/fixtures/test1/local-link.txt new file mode 120000 index 0000000000000..39cd5762dce4e --- /dev/null +++ b/packages/@aws-cdk/assets/test/fs/fixtures/test1/local-link.txt @@ -0,0 +1 @@ +file1.txt \ No newline at end of file diff --git a/packages/@aws-cdk/assets/test/fs/fixtures/test1/subdir/file2.txt b/packages/@aws-cdk/assets/test/fs/fixtures/test1/subdir/file2.txt new file mode 100644 index 0000000000000..97bbbd35efdff --- /dev/null +++ b/packages/@aws-cdk/assets/test/fs/fixtures/test1/subdir/file2.txt @@ -0,0 +1 @@ +file2 in subdir \ No newline at end of file diff --git a/packages/@aws-cdk/assets/test/fs/fixtures/test1/subdir2/empty-subdir/.hidden b/packages/@aws-cdk/assets/test/fs/fixtures/test1/subdir2/empty-subdir/.hidden new file mode 100644 index 0000000000000..b96b7256c6541 --- /dev/null +++ b/packages/@aws-cdk/assets/test/fs/fixtures/test1/subdir2/empty-subdir/.hidden @@ -0,0 +1 @@ +hidden file \ No newline at end of file diff --git a/packages/@aws-cdk/assets/test/fs/fixtures/test1/subdir2/subdir3/file3.txt b/packages/@aws-cdk/assets/test/fs/fixtures/test1/subdir2/subdir3/file3.txt new file mode 100644 index 0000000000000..eae5e936a040d --- /dev/null +++ b/packages/@aws-cdk/assets/test/fs/fixtures/test1/subdir2/subdir3/file3.txt @@ -0,0 +1 @@ +file3 in subdir2/subdir3 diff --git a/packages/@aws-cdk/assets/test/fs/test.fs-copy.ts b/packages/@aws-cdk/assets/test/fs/test.fs-copy.ts new file mode 100644 index 0000000000000..ac0693d70d361 --- /dev/null +++ b/packages/@aws-cdk/assets/test/fs/test.fs-copy.ts @@ -0,0 +1,146 @@ +import fs = require('fs'); +import { Test } from 'nodeunit'; +import os = require('os'); +import path = require('path'); +import { copyDirectory } from '../../lib/fs/copy'; +import { FollowMode } from '../../lib/fs/follow-mode'; + +export = { + 'Default: copies all files and subdirectories, with default follow mode is "External"'(test: Test) { + // GIVEN + const outdir = fs.mkdtempSync(path.join(os.tmpdir(), 'copy-tests')); + + // WHEN + copyDirectory(path.join(__dirname, 'fixtures', 'test1'), outdir); + + // THEN + test.deepEqual(tree(outdir), [ + 'external-link.txt', + 'file1.txt', + 'local-link.txt => file1.txt', + 'subdir (D)', + ' file2.txt', + 'subdir2 (D)', + ' empty-subdir (D)', + ' .hidden', + ' subdir3 (D)', + ' file3.txt' + ]); + test.done(); + }, + + 'Always: follow all symlinks'(test: Test) { + // GIVEN + const outdir = fs.mkdtempSync(path.join(os.tmpdir(), 'copy-tests')); + + // WHEN + copyDirectory(path.join(__dirname, 'fixtures', 'symlinks'), outdir, { + follow: FollowMode.Always + }); + + // THEN + test.deepEqual(tree(outdir), [ + 'external-dir-link (D)', + ' file2.txt', + 'external-link.txt', + 'indirect-external-link.txt', + 'local-dir-link (D)', + ' file-in-subdir.txt', + 'local-link.txt', + 'normal-dir (D)', + ' file-in-subdir.txt', + 'normal-file.txt' + ]); + test.done(); + }, + + 'Never: do not follow all symlinks'(test: Test) { + // GIVEN + const outdir = fs.mkdtempSync(path.join(os.tmpdir(), 'copy-tests')); + + // WHEN + copyDirectory(path.join(__dirname, 'fixtures', 'symlinks'), outdir, { + follow: FollowMode.Never + }); + + // THEN + test.deepEqual(tree(outdir), [ + 'external-dir-link => ../test1/subdir', + 'external-link.txt => ../test1/subdir2/subdir3/file3.txt', + 'indirect-external-link.txt => external-link.txt', + 'local-dir-link => normal-dir', + 'local-link.txt => normal-file.txt', + 'normal-dir (D)', + ' file-in-subdir.txt', + 'normal-file.txt' + ]); + test.done(); + }, + + 'External: follow only external symlinks'(test: Test) { + // GIVEN + const outdir = fs.mkdtempSync(path.join(os.tmpdir(), 'copy-tests')); + + // WHEN + copyDirectory(path.join(__dirname, 'fixtures', 'symlinks'), outdir, { + follow: FollowMode.External + }); + + // THEN + test.deepEqual(tree(outdir), [ + 'external-dir-link (D)', + ' file2.txt', + 'external-link.txt', + 'indirect-external-link.txt => external-link.txt', + 'local-dir-link => normal-dir', + 'local-link.txt => normal-file.txt', + 'normal-dir (D)', + ' file-in-subdir.txt', + 'normal-file.txt' + ]); + + test.done(); + }, + + 'exclude'(test: Test) { + // GIVEN + const outdir = fs.mkdtempSync(path.join(os.tmpdir(), 'copy-tests')); + + // WHEN + copyDirectory(path.join(__dirname, 'fixtures', 'test1'), outdir, { + exclude: [ + '*', + '!subdir2', + '!subdir2/**/*', + '.*' + ] + }); + + // THEN + test.deepEqual(tree(outdir), [ + 'subdir2 (D)', + ' empty-subdir (D)', + ' subdir3 (D)', + ' file3.txt' + ]); + test.done(); + }, +}; + +function tree(dir: string, depth = ''): string[] { + const lines = []; + for (const file of fs.readdirSync(dir).sort()) { + const filePath = path.join(dir, file); + const stat = fs.lstatSync(filePath); + if (stat.isSymbolicLink()) { + const linkDest = fs.readlinkSync(filePath); + lines.push(depth + file + ' => ' + linkDest); + } else if (stat.isDirectory()) { + lines.push(depth + file + ' (D)'); + lines.push(...tree(filePath, depth + ' ')); + } else { + lines.push(depth + file); + } + } + return lines; +} \ No newline at end of file diff --git a/packages/@aws-cdk/assets/test/fs/test.fs-fingerprint.ts b/packages/@aws-cdk/assets/test/fs/test.fs-fingerprint.ts new file mode 100644 index 0000000000000..87cf001562055 --- /dev/null +++ b/packages/@aws-cdk/assets/test/fs/test.fs-fingerprint.ts @@ -0,0 +1,108 @@ +import fs = require('fs'); +import { Test } from 'nodeunit'; +import os = require('os'); +import path = require('path'); +import { copyDirectory } from '../../lib/fs/copy'; +import { fingerprint } from '../../lib/fs/fingerprint'; + +export = { + 'single file'(test: Test) { + // GIVEN + const workdir = fs.mkdtempSync(path.join(os.tmpdir(), 'hash-tests')); + const content = 'Hello, world!'; + const input1 = path.join(workdir, 'input1.txt'); + const input2 = path.join(workdir, 'input2.txt'); + const input3 = path.join(workdir, 'input3.txt'); + fs.writeFileSync(input1, content); + fs.writeFileSync(input2, content); + fs.writeFileSync(input3, content + '.'); // add one character, hash should be different + + // WHEN + const hash1 = fingerprint(input1); + const hash2 = fingerprint(input2); + const hash3 = fingerprint(input3); + + // THEN + test.deepEqual(hash1, hash2); + test.notDeepEqual(hash3, hash1); + test.done(); + }, + + 'empty file'(test: Test) { + // GIVEN + const workdir = fs.mkdtempSync(path.join(os.tmpdir(), 'hash-tests')); + const input1 = path.join(workdir, 'empty'); + const input2 = path.join(workdir, 'empty'); + fs.writeFileSync(input1, ''); + fs.writeFileSync(input2, ''); + + // WHEN + const hash1 = fingerprint(input1); + const hash2 = fingerprint(input2); + + // THEN + test.deepEqual(hash1, hash2); + test.done(); + }, + + 'directory'(test: Test) { + // GIVEN + const srcdir = path.join(__dirname, 'fixtures', 'symlinks'); + const outdir = fs.mkdtempSync(path.join(os.tmpdir(), 'copy-tests')); + copyDirectory(srcdir, outdir); + + // WHEN + const hashSrc = fingerprint(srcdir); + const hashCopy = fingerprint(outdir); + + // THEN + test.deepEqual(hashSrc, hashCopy); + test.done(); + }, + + 'directory, rename files (fingerprint should change)'(test: Test) { + // GIVEN + const srcdir = path.join(__dirname, 'fixtures', 'symlinks'); + const cpydir = fs.mkdtempSync(path.join(os.tmpdir(), 'fingerprint-tests')); + copyDirectory(srcdir, cpydir); + + // be careful not to break a symlink + fs.renameSync(path.join(cpydir, 'normal-dir', 'file-in-subdir.txt'), path.join(cpydir, 'move-me.txt')); + + // WHEN + const hashSrc = fingerprint(srcdir); + const hashCopy = fingerprint(cpydir); + + // THEN + test.notDeepEqual(hashSrc, hashCopy); + test.done(); + }, + + 'external symlink content changes (fingerprint should change)'(test: Test) { + // GIVEN + const dir1 = fs.mkdtempSync(path.join(os.tmpdir(), 'fingerprint-tests')); + const dir2 = fs.mkdtempSync(path.join(os.tmpdir(), 'fingerprint-tests')); + const target = path.join(dir1, 'boom.txt'); + const content = 'boom'; + fs.writeFileSync(target, content); + fs.symlinkSync(target, path.join(dir2, 'link-to-boom.txt')); + + // now dir2 contains a symlink to a file in dir1 + + // WHEN + const original = fingerprint(dir2); + + // now change the contents of the target + fs.writeFileSync(target, 'changning you!'); + const afterChange = fingerprint(dir2); + + // revert the content to original and expect hash to be reverted + fs.writeFileSync(target, content); + const afterRevert = fingerprint(dir2); + + // THEN + test.notDeepEqual(original, afterChange); + test.deepEqual(afterRevert, original); + test.done(); + } +}; diff --git a/packages/@aws-cdk/assets/test/test.asset.ts b/packages/@aws-cdk/assets/test/test.asset.ts index 82147d630f893..1d8ad7d56777d 100644 --- a/packages/@aws-cdk/assets/test/test.asset.ts +++ b/packages/@aws-cdk/assets/test/test.asset.ts @@ -1,17 +1,21 @@ import { expect, haveResource, ResourcePart, SynthUtils } from '@aws-cdk/assert'; import iam = require('@aws-cdk/aws-iam'); import cdk = require('@aws-cdk/cdk'); +import { App, Stack } from '@aws-cdk/cdk'; import cxapi = require('@aws-cdk/cx-api'); +import fs = require('fs'); import { Test } from 'nodeunit'; +import os = require('os'); import path = require('path'); import { FileAsset, ZipDirectoryAsset } from '../lib/asset'; +const SAMPLE_ASSET_DIR = path.join(__dirname, 'sample-asset-directory'); + export = { 'simple use case'(test: Test) { const stack = new cdk.Stack(); - const dirPath = path.join(__dirname, 'sample-asset-directory'); const asset = new ZipDirectoryAsset(stack, 'MyAsset', { - path: dirPath + path: SAMPLE_ASSET_DIR }); // verify that metadata contains an "aws:cdk:asset" entry with @@ -19,18 +23,17 @@ export = { const entry = asset.node.metadata.find(m => m.type === 'aws:cdk:asset'); test.ok(entry, 'found metadata entry'); - // console.error(JSON.stringify(stack.node.resolve(entry!.data))); + // verify that now the template contains parameters for this asset + const template = SynthUtils.toCloudFormation(stack); test.deepEqual(stack.node.resolve(entry!.data), { - path: dirPath, + path: SAMPLE_ASSET_DIR, id: 'MyAsset', packaging: 'zip', s3BucketParameter: 'MyAssetS3Bucket68C9B344', s3KeyParameter: 'MyAssetS3VersionKey68E1A45D', }); - // verify that now the template contains parameters for this asset - const template = SynthUtils.toCloudFormation(stack); test.equal(template.Parameters.MyAssetS3Bucket68C9B344.Type, 'String'); test.equal(template.Parameters.MyAssetS3VersionKey68E1A45D.Type, 'String'); @@ -65,6 +68,10 @@ export = { const asset = new FileAsset(stack, 'MyAsset', { path: filePath }); const entry = asset.node.metadata.find(m => m.type === 'aws:cdk:asset'); test.ok(entry, 'found metadata entry'); + + // synthesize first so "prepare" is called + const template = SynthUtils.toCloudFormation(stack); + test.deepEqual(stack.node.resolve(entry!.data), { path: filePath, packaging: 'file', @@ -74,7 +81,6 @@ export = { }); // verify that now the template contains parameters for this asset - const template = SynthUtils.toCloudFormation(stack); test.equal(template.Parameters.MyAssetS3Bucket68C9B344.Type, 'String'); test.equal(template.Parameters.MyAssetS3VersionKey68E1A45D.Type, 'String'); @@ -194,9 +200,8 @@ export = { // GIVEN const stack = new cdk.Stack(); - const location = path.join(__dirname, 'sample-asset-directory'); const resource = new cdk.CfnResource(stack, 'MyResource', { type: 'My::Resource::Type' }); - const asset = new ZipDirectoryAsset(stack, 'MyAsset', { path: location }); + const asset = new ZipDirectoryAsset(stack, 'MyAsset', { path: SAMPLE_ASSET_DIR }); // WHEN asset.addResourceMetadata(resource, 'PropName'); @@ -204,11 +209,155 @@ export = { // THEN expect(stack).notTo(haveResource('My::Resource::Type', { Metadata: { - "aws:asset:path": location, + "aws:asset:path": SAMPLE_ASSET_DIR, "aws:asset:property": "PropName" } }, ResourcePart.CompleteDefinition)); test.done(); + }, + + 'staging': { + + 'copy file assets under .assets/fingerprint.ext'(test: Test) { + const tempdir = mkdtempSync(); + process.chdir(tempdir); // change current directory to somewhere in /tmp + + // GIVEN + const app = new App({ + context: { [cxapi.ASSET_STAGING_DIR_CONTEXT]: '.assets' } + }); + const stack = new Stack(app, 'stack'); + + // WHEN + new FileAsset(stack, 'ZipFile', { + path: path.join(SAMPLE_ASSET_DIR, 'sample-zip-asset.zip') + }); + + new FileAsset(stack, 'TextFile', { + path: path.join(SAMPLE_ASSET_DIR, 'sample-asset-file.txt') + }); + + // THEN + app.run(); + test.ok(fs.existsSync(path.join(tempdir, '.assets'))); + test.ok(fs.existsSync(path.join(tempdir, '.assets', 'fdb4701ff6c99e676018ee2c24a3119b.zip'))); + fs.readdirSync(path.join(tempdir, '.assets')); + test.done(); + }, + + 'copy directory under .assets/fingerprint/**'(test: Test) { + const tempdir = mkdtempSync(); + process.chdir(tempdir); // change current directory to somewhere in /tmp + + // GIVEN + const app = new App({ + context: { [cxapi.ASSET_STAGING_DIR_CONTEXT]: '.assets' } + }); + const stack = new Stack(app, 'stack'); + + // WHEN + new ZipDirectoryAsset(stack, 'ZipDirectory', { + path: SAMPLE_ASSET_DIR + }); + + // THEN + app.run(); + test.ok(fs.existsSync(path.join(tempdir, '.assets'))); + test.ok(fs.existsSync(path.join(tempdir, '.assets', 'b550524e103eb4cf257c594fba5b9fe8', 'sample-asset-file.txt'))); + test.ok(fs.existsSync(path.join(tempdir, '.assets', 'b550524e103eb4cf257c594fba5b9fe8', 'sample-jar-asset.jar'))); + fs.readdirSync(path.join(tempdir, '.assets')); + test.done(); + }, + + 'staging path is relative if the dir is below the working directory'(test: Test) { + // GIVEN + const tempdir = mkdtempSync(); + process.chdir(tempdir); // change current directory to somewhere in /tmp + + const staging = '.my-awesome-staging-directory'; + const app = new App({ + context: { + [cxapi.ASSET_STAGING_DIR_CONTEXT]: staging, + [cxapi.ASSET_RESOURCE_METADATA_ENABLED_CONTEXT]: 'true', + } + }); + + const stack = new Stack(app, 'stack'); + + const resource = new cdk.CfnResource(stack, 'MyResource', { type: 'My::Resource::Type' }); + const asset = new ZipDirectoryAsset(stack, 'MyAsset', { path: SAMPLE_ASSET_DIR }); + + // WHEN + asset.addResourceMetadata(resource, 'PropName'); + + const session = app.run(); + const template = SynthUtils.templateForStackName(session, stack.name); + + test.deepEqual(template.Resources.MyResource.Metadata, { + "aws:asset:path": `.my-awesome-staging-directory/b550524e103eb4cf257c594fba5b9fe8`, + "aws:asset:property": "PropName" + }); + test.done(); + }, + + 'if staging directory is absolute, asset path is absolute'(test: Test) { + // GIVEN + const staging = path.resolve(mkdtempSync()); + const app = new App({ + context: { + [cxapi.ASSET_STAGING_DIR_CONTEXT]: staging, + [cxapi.ASSET_RESOURCE_METADATA_ENABLED_CONTEXT]: 'true', + } + }); + + const stack = new Stack(app, 'stack'); + + const resource = new cdk.CfnResource(stack, 'MyResource', { type: 'My::Resource::Type' }); + const asset = new ZipDirectoryAsset(stack, 'MyAsset', { path: SAMPLE_ASSET_DIR }); + + // WHEN + asset.addResourceMetadata(resource, 'PropName'); + + const session = app.run(); + const template = SynthUtils.templateForStackName(session, stack.name); + + test.deepEqual(template.Resources.MyResource.Metadata, { + "aws:asset:path": `${staging}/b550524e103eb4cf257c594fba5b9fe8`, + "aws:asset:property": "PropName" + }); + test.done(); + }, + + 'cdk metadata points to staged asset'(test: Test) { + // GIVEN + const tempdir = mkdtempSync(); + process.chdir(tempdir); // change current directory to somewhere in /tmp + + const staging = '.stageme'; + + const app = new App({ + context: { + [cxapi.ASSET_STAGING_DIR_CONTEXT]: staging, + } + }); + + const stack = new Stack(app, 'stack'); + + new ZipDirectoryAsset(stack, 'MyAsset', { path: SAMPLE_ASSET_DIR }); + + // WHEN + const session = app.run(); + const artifact = session.getArtifact(stack.name); + + const md = Object.values(artifact.metadata || {})[0][0].data; + test.deepEqual(md.path, '.stageme/b550524e103eb4cf257c594fba5b9fe8'); + test.done(); + } + } }; + +function mkdtempSync() { + return fs.mkdtempSync(path.join(os.tmpdir(), 'test.assets')); +} diff --git a/packages/@aws-cdk/cx-api/lib/cxapi.ts b/packages/@aws-cdk/cx-api/lib/cxapi.ts index ae76ea0ea1eac..459e3bee879e1 100644 --- a/packages/@aws-cdk/cx-api/lib/cxapi.ts +++ b/packages/@aws-cdk/cx-api/lib/cxapi.ts @@ -177,3 +177,9 @@ export const OUTFILE_NAME = 'cdk.out'; * Disable the collection and reporting of version information. */ export const DISABLE_VERSION_REPORTING = 'aws:cdk:disable-version-reporting'; + +/** + * If this context key is set, the CDK will stage assets under the specified + * directory. Otherwise, assets will not be staged. + */ +export const ASSET_STAGING_DIR_CONTEXT = 'aws:cdk:asset-staging-dir'; diff --git a/packages/aws-cdk/bin/cdk.ts b/packages/aws-cdk/bin/cdk.ts index 604a75044ab2a..02e7a78681950 100644 --- a/packages/aws-cdk/bin/cdk.ts +++ b/packages/aws-cdk/bin/cdk.ts @@ -50,6 +50,7 @@ async function parseCommandLineArguments() { .option('asset-metadata', { type: 'boolean', desc: 'Include "aws:asset:*" CloudFormation metadata for resources that user assets (enabled by default)', default: true }) .option('role-arn', { type: 'string', alias: 'r', desc: 'ARN of Role to use when invoking CloudFormation', default: undefined, requiresArg: true }) .option('toolkit-stack-name', { type: 'string', desc: 'The name of the CDK toolkit stack', requiresArg: true }) + .option('staging', { type: 'string', desc: 'directory name for staging assets (use --no-asset-staging to disable)', default: '.cdk.staging' }) .command([ 'list', 'ls' ], 'Lists all stacks in the app', yargs => yargs .option('long', { type: 'boolean', default: false, alias: 'l', desc: 'display environment information for each stack' })) .command([ 'synthesize [STACKS..]', 'synth [STACKS..]' ], 'Synthesizes and prints the CloudFormation template for this stack', yargs => yargs diff --git a/packages/aws-cdk/lib/api/cxapp/exec.ts b/packages/aws-cdk/lib/api/cxapp/exec.ts index 5d830301dfd1d..72d1412c367c6 100644 --- a/packages/aws-cdk/lib/api/cxapp/exec.ts +++ b/packages/aws-cdk/lib/api/cxapp/exec.ts @@ -42,6 +42,9 @@ export async function execProgram(aws: SDK, config: Configuration): Promise(); + args.push('--no-path-metadata'); + args.push('--no-asset-metadata'); + args.push('--no-staging'); + + const actual = await test.invoke(['--json', ...args, 'synth'], { json: true, context: STATIC_TEST_CONTEXT }); const diff = diffTemplate(expected, actual); diff --git a/tools/cdk-integ-tools/bin/cdk-integ.ts b/tools/cdk-integ-tools/bin/cdk-integ.ts index 96c39794d5c0f..8eed72579a672 100644 --- a/tools/cdk-integ-tools/bin/cdk-integ.ts +++ b/tools/cdk-integ-tools/bin/cdk-integ.ts @@ -28,6 +28,7 @@ async function main() { // don't inject cloudformation metadata into template args.push('--no-path-metadata'); args.push('--no-asset-metadata'); + args.push('--no-staging'); // inject "--verbose" to the command line of "cdk" if we are in verbose mode if (argv.verbose) {