From 96aa88b2175be7daf663f9acdd812d4da3121922 Mon Sep 17 00:00:00 2001 From: Elad Ben-Israel Date: Thu, 4 Apr 2019 23:00:29 +0300 Subject: [PATCH] feat: stage assets under .cdk.assets To ensure that assets are available for the toolchain to deploy after the CDK app exists, the CLI will, by default, request that the app will stage the assets under the `.cdk.assets` directory (relative to working directory). The CDK will then *copy* all assets from their source locations to this staging directory and will refer to the staging location as the asset path. Assets will be stored using their content fingerprint (md5 hash) so they will never be copied twice unless they change. Fixes #1716 TODO: - [ ] docker assets - [ ] toolkit support - [ ] toolkit test --- packages/@aws-cdk/assets/lib/asset.ts | 85 +++++++++- packages/@aws-cdk/assets/lib/fs/copy.ts | 89 +++++++++++ .../@aws-cdk/assets/lib/fs/fingerprint.ts | 86 +++++++++++ .../@aws-cdk/assets/lib/fs/follow-mode.ts | 29 ++++ packages/@aws-cdk/assets/lib/fs/index.ts | 3 + packages/@aws-cdk/assets/package-lock.json | 41 +++++ packages/@aws-cdk/assets/package.json | 11 +- .../fs/fixtures/symlinks/external-dir-link | 1 + .../fs/fixtures/symlinks/external-link.txt | 1 + .../symlinks/indirect-external-link.txt | 1 + .../test/fs/fixtures/symlinks/local-dir-link | 1 + .../test/fs/fixtures/symlinks/local-link.txt | 1 + .../symlinks/normal-dir/file-in-subdir.txt | 1 + .../test/fs/fixtures/symlinks/normal-file.txt | 1 + .../test/fs/fixtures/test1/external-link.txt | 1 + .../assets/test/fs/fixtures/test1/file1.txt | 1 + .../test/fs/fixtures/test1/local-link.txt | 1 + .../test/fs/fixtures/test1/subdir/file2.txt | 1 + .../test1/subdir2/empty-subdir/.hidden | 1 + .../fixtures/test1/subdir2/subdir3/file3.txt | 1 + .../@aws-cdk/assets/test/fs/test.fs-copy.ts | 146 ++++++++++++++++++ .../assets/test/fs/test.fs-fingerprint.ts | 108 +++++++++++++ packages/@aws-cdk/assets/test/test.asset.ts | 140 +++++++++++++++-- packages/@aws-cdk/cx-api/lib/cxapi.ts | 6 + 24 files changed, 740 insertions(+), 17 deletions(-) create mode 100644 packages/@aws-cdk/assets/lib/fs/copy.ts create mode 100644 packages/@aws-cdk/assets/lib/fs/fingerprint.ts create mode 100644 packages/@aws-cdk/assets/lib/fs/follow-mode.ts create mode 100644 packages/@aws-cdk/assets/lib/fs/index.ts create mode 100644 packages/@aws-cdk/assets/package-lock.json create mode 120000 packages/@aws-cdk/assets/test/fs/fixtures/symlinks/external-dir-link create mode 120000 packages/@aws-cdk/assets/test/fs/fixtures/symlinks/external-link.txt create mode 120000 packages/@aws-cdk/assets/test/fs/fixtures/symlinks/indirect-external-link.txt create mode 120000 packages/@aws-cdk/assets/test/fs/fixtures/symlinks/local-dir-link create mode 120000 packages/@aws-cdk/assets/test/fs/fixtures/symlinks/local-link.txt create mode 100644 packages/@aws-cdk/assets/test/fs/fixtures/symlinks/normal-dir/file-in-subdir.txt create mode 100644 packages/@aws-cdk/assets/test/fs/fixtures/symlinks/normal-file.txt create mode 120000 packages/@aws-cdk/assets/test/fs/fixtures/test1/external-link.txt create mode 100644 packages/@aws-cdk/assets/test/fs/fixtures/test1/file1.txt create mode 120000 packages/@aws-cdk/assets/test/fs/fixtures/test1/local-link.txt create mode 100644 packages/@aws-cdk/assets/test/fs/fixtures/test1/subdir/file2.txt create mode 100644 packages/@aws-cdk/assets/test/fs/fixtures/test1/subdir2/empty-subdir/.hidden create mode 100644 packages/@aws-cdk/assets/test/fs/fixtures/test1/subdir2/subdir3/file3.txt create mode 100644 packages/@aws-cdk/assets/test/fs/test.fs-copy.ts create mode 100644 packages/@aws-cdk/assets/test/fs/test.fs-fingerprint.ts diff --git a/packages/@aws-cdk/assets/lib/asset.ts b/packages/@aws-cdk/assets/lib/asset.ts index d985af72954b7..f42afad7b4064 100644 --- a/packages/@aws-cdk/assets/lib/asset.ts +++ b/packages/@aws-cdk/assets/lib/asset.ts @@ -4,6 +4,7 @@ import cdk = require('@aws-cdk/cdk'); import cxapi = require('@aws-cdk/cx-api'); import fs = require('fs'); import path = require('path'); +import { copyDirectory, fingerprint } from './fs'; /** * Defines the way an asset is packaged before it is uploaded to S3. @@ -61,7 +62,10 @@ export class Asset extends cdk.Construct { public readonly s3Url: string; /** - * Resolved full-path location of this asset. + * The path to the asset (stringinfied token). + * + * If asset staging is disabled, this will just be the original path. + * If asset staging is enabled it will be the staged path. */ public readonly assetPath: string; @@ -81,19 +85,55 @@ export class Asset extends cdk.Construct { */ private readonly s3Prefix: string; + /** + * Type of asset packaging (file/zip-dir);. + */ + private readonly packaging: AssetPackaging; + + /** + * The path of the asset as it was referenced by the user. + */ + private readonly sourcePath: string; + + /** + * If this is defined, assets will be staged (copied) under this directory. + * Otherwise, they will just be consumed from their original location. + * + * This is controlled by the context key 'aws:cdk:assets-staging-dir' and + * enabled by the CLI by default in order to ensure that when the CDK app + * exists, all assets are available for deployment. Otherwise, if an app + * references assets in temporary locations, those will not be available when + * it exists (see https://github.com/awslabs/aws-cdk/issues/1716). + */ + private readonly stagingDir?: string; + + /** + * The asset path after "prepare" is called. + * + * If staging is disabled, this will just be the original path. + * If staging is enabled it will be the staged path. + */ + private _preparedAssetPath?: string; + constructor(scope: cdk.Construct, id: string, props: GenericAssetProps) { super(scope, id); + this.packaging = props.packaging; + + this.stagingDir = this.node.getContext(cxapi.ASSETS_STAGING_DIR_CONTEXT); + // resolve full path - this.assetPath = path.resolve(props.path); + this.sourcePath = path.resolve(props.path); // sets isZipArchive based on the type of packaging and file extension const allowedExtensions: string[] = ['.jar', '.zip']; this.isZipArchive = props.packaging === AssetPackaging.ZipDirectory ? true - : allowedExtensions.some(ext => this.assetPath.toLowerCase().endsWith(ext)); + : allowedExtensions.some(ext => this.sourcePath.toLowerCase().endsWith(ext)); + + validateAssetOnDisk(this.sourcePath, props.packaging); - validateAssetOnDisk(this.assetPath, props.packaging); + this.assetPath = new cdk.Token(() => this._preparedAssetPath).toString(); // add parameters for s3 bucket and s3 key. those will be set by // the toolkit or by CI/CD when the stack is deployed and will include @@ -178,6 +218,43 @@ export class Asset extends cdk.Construct { // when deploying a new version. this.bucket.grantRead(grantee, `${this.s3Prefix}*`); } + + public prepare() { + const stagingDir = this.stagingDir; + if (!stagingDir) { + this._preparedAssetPath = this.sourcePath; + return; + } + + if (!fs.existsSync(stagingDir)) { + fs.mkdirSync(stagingDir); + } + + const hash = fingerprint(this.sourcePath); + const targetPath = path.join(stagingDir, hash + path.extname(this.sourcePath)); + + this._preparedAssetPath = targetPath; + + // nothing to do if we already have the asset there + if (fs.existsSync(targetPath)) { + return; + } + + // copy file/directory to .assets + switch (this.packaging) { + case AssetPackaging.File: + fs.copyFileSync(this.sourcePath, targetPath); + break; + + case AssetPackaging.ZipDirectory: + fs.mkdirSync(targetPath); + copyDirectory(this.sourcePath, targetPath); + break; + + default: + throw new Error(`Unknown asset packaging type: ${this.packaging}`); + } + } } export interface FileAssetProps { diff --git a/packages/@aws-cdk/assets/lib/fs/copy.ts b/packages/@aws-cdk/assets/lib/fs/copy.ts new file mode 100644 index 0000000000000..6ea1f2a6e5f8c --- /dev/null +++ b/packages/@aws-cdk/assets/lib/fs/copy.ts @@ -0,0 +1,89 @@ +import fs = require('fs'); +import minimatch = require('minimatch'); +import path = require('path'); +import { FollowMode } from './follow-mode'; + +export interface CopyOptions { + /** + * @default External only follows symlinks that are external to the source directory + */ + follow?: FollowMode; + + /** + * glob patterns to exclude from the copy. + */ + exclude?: string[]; +} + +export function copyDirectory(srcDir: string, destDir: string, options: CopyOptions = { }, rootDir?: string) { + const follow = options.follow !== undefined ? options.follow : FollowMode.External; + const exclude = options.exclude || []; + + rootDir = rootDir || srcDir; + + if (!fs.statSync(srcDir).isDirectory()) { + throw new Error(`${srcDir} is not a directory`); + } + + const files = fs.readdirSync(srcDir); + for (const file of files) { + const sourceFilePath = path.join(srcDir, file); + + if (shouldExclude(path.relative(rootDir, sourceFilePath))) { + continue; + } + + const destFilePath = path.join(destDir, file); + + let stat: fs.Stats | undefined = follow === FollowMode.Always + ? fs.statSync(sourceFilePath) + : fs.lstatSync(sourceFilePath); + + if (stat && stat.isSymbolicLink()) { + const target = fs.readlinkSync(sourceFilePath); + + // determine if this is an external link (i.e. the target's absolute path + // is outside of the root directory). + const targetPath = path.normalize(path.resolve(srcDir, target)); + const rootPath = path.normalize(rootDir); + const external = !targetPath.startsWith(rootPath); + + if (follow === FollowMode.External && external) { + stat = fs.statSync(sourceFilePath); + } else { + fs.symlinkSync(target, destFilePath); + stat = undefined; + } + } + + if (stat && stat.isDirectory()) { + fs.mkdirSync(destFilePath); + copyDirectory(sourceFilePath, destFilePath, options, rootDir); + stat = undefined; + } + + if (stat && stat.isFile()) { + fs.copyFileSync(sourceFilePath, destFilePath); + stat = undefined; + } + } + + function shouldExclude(filePath: string): boolean { + let excludeOutput = false; + + for (const pattern of exclude) { + const negate = pattern.startsWith('!'); + const match = minimatch(filePath, pattern, { matchBase: true, flipNegate: true }); + + if (!negate && match) { + excludeOutput = true; + } + + if (negate && match) { + excludeOutput = false; + } + } + + return excludeOutput; + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/assets/lib/fs/fingerprint.ts b/packages/@aws-cdk/assets/lib/fs/fingerprint.ts new file mode 100644 index 0000000000000..06cdb6a0ed2aa --- /dev/null +++ b/packages/@aws-cdk/assets/lib/fs/fingerprint.ts @@ -0,0 +1,86 @@ +import crypto = require('crypto'); +import fs = require('fs'); +import path = require('path'); +import { FollowMode } from './follow-mode'; + +const BUFFER_SIZE = 8 * 1024; + +export interface FingerprintOptions { + /** + * Extra information to encode into the fingerprint (e.g. build instructions + * and other inputs) + */ + extra?: string; + + /** + * List of exclude patterns (see `CopyOptions`) + * @default include all files + */ + exclude?: string[]; + + /** + * What to do when we encounter symlinks. + * @default External only follows symlinks that are external to the source + * directory + */ + follow?: FollowMode; +} + +/** + * Produces fingerprint based on the contents of a single file or an entire directory tree. + * + * The fingerprint will also include: + * 1. An extra string if defined in `options.extra`. + * 2. The set of exclude patterns, if defined in `options.exclude` + * 3. The symlink follow mode value. + * + * @param fileOrDirectory The directory or file to fingerprint + * @param options Fingerprinting options + */ +export function fingerprint(fileOrDirectory: string, options: FingerprintOptions = { }) { + const follow = options.follow !== undefined ? options.follow : FollowMode.External; + const hash = crypto.createHash('md5'); + addToHash(fileOrDirectory); + + hash.update(`==follow==${follow}==\n\n`); + + if (options.extra) { + hash.update(`==extra==${options.extra}==\n\n`); + } + + for (const ex of options.exclude || []) { + hash.update(`==exclude==${ex}==\n\n`); + } + + return hash.digest('hex'); + + function addToHash(pathToAdd: string) { + hash.update('==\n'); + const relativePath = path.relative(fileOrDirectory, pathToAdd); + hash.update(relativePath + '\n'); + hash.update('~~~~~~~~~~~~~~~~~~\n'); + const stat = fs.statSync(pathToAdd); + + if (stat.isSymbolicLink()) { + const target = fs.readlinkSync(pathToAdd); + hash.update(target); + } else if (stat.isDirectory()) { + for (const file of fs.readdirSync(pathToAdd)) { + addToHash(path.join(pathToAdd, file)); + } + } else { + const file = fs.openSync(pathToAdd, 'r'); + const buffer = Buffer.alloc(BUFFER_SIZE); + + try { + let bytesRead; + do { + bytesRead = fs.readSync(file, buffer, 0, BUFFER_SIZE, null); + hash.update(buffer.slice(0, bytesRead)); + } while (bytesRead === BUFFER_SIZE); + } finally { + fs.closeSync(file); + } + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/assets/lib/fs/follow-mode.ts b/packages/@aws-cdk/assets/lib/fs/follow-mode.ts new file mode 100644 index 0000000000000..02ecebfaaa0a7 --- /dev/null +++ b/packages/@aws-cdk/assets/lib/fs/follow-mode.ts @@ -0,0 +1,29 @@ +export enum FollowMode { + /** + * Never follow symlinks. + */ + Never = 'never', + + /** + * Materialize all symlinks, whether they are internal or external to the source directory. + */ + Always = 'always', + + /** + * Only follows symlinks that are external to the source directory. + */ + External = 'external', + + // ----------------- TODO:::::::::::::::::::::::::::::::::::::::::::: + /** + * Forbids source from having any symlinks pointing outside of the source + * tree. + * + * This is the safest mode of operation as it ensures that copy operations + * won't materialize files from the user's file system. Internal symlinks are + * not followed. + * + * If the copy operation runs into an external symlink, it will fail. + */ + BlockExternal = 'internal-only', +} \ No newline at end of file diff --git a/packages/@aws-cdk/assets/lib/fs/index.ts b/packages/@aws-cdk/assets/lib/fs/index.ts new file mode 100644 index 0000000000000..31b1f468bbdfc --- /dev/null +++ b/packages/@aws-cdk/assets/lib/fs/index.ts @@ -0,0 +1,3 @@ +export * from './fingerprint'; +export * from './follow-mode'; +export * from './copy'; \ No newline at end of file diff --git a/packages/@aws-cdk/assets/package-lock.json b/packages/@aws-cdk/assets/package-lock.json new file mode 100644 index 0000000000000..d3b41e065d7a8 --- /dev/null +++ b/packages/@aws-cdk/assets/package-lock.json @@ -0,0 +1,41 @@ +{ + "name": "@aws-cdk/assets", + "version": "0.28.0", + "lockfileVersion": 1, + "requires": true, + "dependencies": { + "@types/minimatch": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.3.tgz", + "integrity": "sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA==", + "dev": true + }, + "balanced-match": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", + "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" + }, + "brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "requires": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" + }, + "minimatch": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", + "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "requires": { + "brace-expansion": "^1.1.7" + } + } + } +} diff --git a/packages/@aws-cdk/assets/package.json b/packages/@aws-cdk/assets/package.json index d2d012b51d7a9..1c9f8c041d0e3 100644 --- a/packages/@aws-cdk/assets/package.json +++ b/packages/@aws-cdk/assets/package.json @@ -55,6 +55,7 @@ "license": "Apache-2.0", "devDependencies": { "@aws-cdk/assert": "^0.28.0", + "@types/minimatch": "^3.0.3", "aws-cdk": "^0.28.0", "cdk-build-tools": "^0.28.0", "cdk-integ-tools": "^0.28.0", @@ -64,7 +65,8 @@ "@aws-cdk/aws-iam": "^0.28.0", "@aws-cdk/aws-s3": "^0.28.0", "@aws-cdk/cdk": "^0.28.0", - "@aws-cdk/cx-api": "^0.28.0" + "@aws-cdk/cx-api": "^0.28.0", + "minimatch": "^3.0.4" }, "homepage": "https://github.com/awslabs/aws-cdk", "peerDependencies": { @@ -74,5 +76,8 @@ }, "engines": { "node": ">= 8.10.0" - } -} \ No newline at end of file + }, + "bundleDependencies": [ + "minimatch" + ] +} diff --git a/packages/@aws-cdk/assets/test/fs/fixtures/symlinks/external-dir-link b/packages/@aws-cdk/assets/test/fs/fixtures/symlinks/external-dir-link new file mode 120000 index 0000000000000..b9447033a4279 --- /dev/null +++ b/packages/@aws-cdk/assets/test/fs/fixtures/symlinks/external-dir-link @@ -0,0 +1 @@ +../test1/subdir \ No newline at end of file diff --git a/packages/@aws-cdk/assets/test/fs/fixtures/symlinks/external-link.txt b/packages/@aws-cdk/assets/test/fs/fixtures/symlinks/external-link.txt new file mode 120000 index 0000000000000..267020c936652 --- /dev/null +++ b/packages/@aws-cdk/assets/test/fs/fixtures/symlinks/external-link.txt @@ -0,0 +1 @@ +../test1/subdir2/subdir3/file3.txt \ No newline at end of file diff --git a/packages/@aws-cdk/assets/test/fs/fixtures/symlinks/indirect-external-link.txt b/packages/@aws-cdk/assets/test/fs/fixtures/symlinks/indirect-external-link.txt new file mode 120000 index 0000000000000..907a2a65b1515 --- /dev/null +++ b/packages/@aws-cdk/assets/test/fs/fixtures/symlinks/indirect-external-link.txt @@ -0,0 +1 @@ +external-link.txt \ No newline at end of file diff --git a/packages/@aws-cdk/assets/test/fs/fixtures/symlinks/local-dir-link b/packages/@aws-cdk/assets/test/fs/fixtures/symlinks/local-dir-link new file mode 120000 index 0000000000000..42101049ac31a --- /dev/null +++ b/packages/@aws-cdk/assets/test/fs/fixtures/symlinks/local-dir-link @@ -0,0 +1 @@ +normal-dir \ No newline at end of file diff --git a/packages/@aws-cdk/assets/test/fs/fixtures/symlinks/local-link.txt b/packages/@aws-cdk/assets/test/fs/fixtures/symlinks/local-link.txt new file mode 120000 index 0000000000000..b3c6fdbd8bfad --- /dev/null +++ b/packages/@aws-cdk/assets/test/fs/fixtures/symlinks/local-link.txt @@ -0,0 +1 @@ +normal-file.txt \ No newline at end of file diff --git a/packages/@aws-cdk/assets/test/fs/fixtures/symlinks/normal-dir/file-in-subdir.txt b/packages/@aws-cdk/assets/test/fs/fixtures/symlinks/normal-dir/file-in-subdir.txt new file mode 100644 index 0000000000000..f52de026412b3 --- /dev/null +++ b/packages/@aws-cdk/assets/test/fs/fixtures/symlinks/normal-dir/file-in-subdir.txt @@ -0,0 +1 @@ +file in subdir diff --git a/packages/@aws-cdk/assets/test/fs/fixtures/symlinks/normal-file.txt b/packages/@aws-cdk/assets/test/fs/fixtures/symlinks/normal-file.txt new file mode 100644 index 0000000000000..d627587e36e77 --- /dev/null +++ b/packages/@aws-cdk/assets/test/fs/fixtures/symlinks/normal-file.txt @@ -0,0 +1 @@ +this is a normal file diff --git a/packages/@aws-cdk/assets/test/fs/fixtures/test1/external-link.txt b/packages/@aws-cdk/assets/test/fs/fixtures/test1/external-link.txt new file mode 120000 index 0000000000000..c7ba61290b25a --- /dev/null +++ b/packages/@aws-cdk/assets/test/fs/fixtures/test1/external-link.txt @@ -0,0 +1 @@ +../symlinks/normal-file.txt \ No newline at end of file diff --git a/packages/@aws-cdk/assets/test/fs/fixtures/test1/file1.txt b/packages/@aws-cdk/assets/test/fs/fixtures/test1/file1.txt new file mode 100644 index 0000000000000..e2129701f1a4d --- /dev/null +++ b/packages/@aws-cdk/assets/test/fs/fixtures/test1/file1.txt @@ -0,0 +1 @@ +file1 diff --git a/packages/@aws-cdk/assets/test/fs/fixtures/test1/local-link.txt b/packages/@aws-cdk/assets/test/fs/fixtures/test1/local-link.txt new file mode 120000 index 0000000000000..39cd5762dce4e --- /dev/null +++ b/packages/@aws-cdk/assets/test/fs/fixtures/test1/local-link.txt @@ -0,0 +1 @@ +file1.txt \ No newline at end of file diff --git a/packages/@aws-cdk/assets/test/fs/fixtures/test1/subdir/file2.txt b/packages/@aws-cdk/assets/test/fs/fixtures/test1/subdir/file2.txt new file mode 100644 index 0000000000000..97bbbd35efdff --- /dev/null +++ b/packages/@aws-cdk/assets/test/fs/fixtures/test1/subdir/file2.txt @@ -0,0 +1 @@ +file2 in subdir \ No newline at end of file diff --git a/packages/@aws-cdk/assets/test/fs/fixtures/test1/subdir2/empty-subdir/.hidden b/packages/@aws-cdk/assets/test/fs/fixtures/test1/subdir2/empty-subdir/.hidden new file mode 100644 index 0000000000000..b96b7256c6541 --- /dev/null +++ b/packages/@aws-cdk/assets/test/fs/fixtures/test1/subdir2/empty-subdir/.hidden @@ -0,0 +1 @@ +hidden file \ No newline at end of file diff --git a/packages/@aws-cdk/assets/test/fs/fixtures/test1/subdir2/subdir3/file3.txt b/packages/@aws-cdk/assets/test/fs/fixtures/test1/subdir2/subdir3/file3.txt new file mode 100644 index 0000000000000..eae5e936a040d --- /dev/null +++ b/packages/@aws-cdk/assets/test/fs/fixtures/test1/subdir2/subdir3/file3.txt @@ -0,0 +1 @@ +file3 in subdir2/subdir3 diff --git a/packages/@aws-cdk/assets/test/fs/test.fs-copy.ts b/packages/@aws-cdk/assets/test/fs/test.fs-copy.ts new file mode 100644 index 0000000000000..ac0693d70d361 --- /dev/null +++ b/packages/@aws-cdk/assets/test/fs/test.fs-copy.ts @@ -0,0 +1,146 @@ +import fs = require('fs'); +import { Test } from 'nodeunit'; +import os = require('os'); +import path = require('path'); +import { copyDirectory } from '../../lib/fs/copy'; +import { FollowMode } from '../../lib/fs/follow-mode'; + +export = { + 'Default: copies all files and subdirectories, with default follow mode is "External"'(test: Test) { + // GIVEN + const outdir = fs.mkdtempSync(path.join(os.tmpdir(), 'copy-tests')); + + // WHEN + copyDirectory(path.join(__dirname, 'fixtures', 'test1'), outdir); + + // THEN + test.deepEqual(tree(outdir), [ + 'external-link.txt', + 'file1.txt', + 'local-link.txt => file1.txt', + 'subdir (D)', + ' file2.txt', + 'subdir2 (D)', + ' empty-subdir (D)', + ' .hidden', + ' subdir3 (D)', + ' file3.txt' + ]); + test.done(); + }, + + 'Always: follow all symlinks'(test: Test) { + // GIVEN + const outdir = fs.mkdtempSync(path.join(os.tmpdir(), 'copy-tests')); + + // WHEN + copyDirectory(path.join(__dirname, 'fixtures', 'symlinks'), outdir, { + follow: FollowMode.Always + }); + + // THEN + test.deepEqual(tree(outdir), [ + 'external-dir-link (D)', + ' file2.txt', + 'external-link.txt', + 'indirect-external-link.txt', + 'local-dir-link (D)', + ' file-in-subdir.txt', + 'local-link.txt', + 'normal-dir (D)', + ' file-in-subdir.txt', + 'normal-file.txt' + ]); + test.done(); + }, + + 'Never: do not follow all symlinks'(test: Test) { + // GIVEN + const outdir = fs.mkdtempSync(path.join(os.tmpdir(), 'copy-tests')); + + // WHEN + copyDirectory(path.join(__dirname, 'fixtures', 'symlinks'), outdir, { + follow: FollowMode.Never + }); + + // THEN + test.deepEqual(tree(outdir), [ + 'external-dir-link => ../test1/subdir', + 'external-link.txt => ../test1/subdir2/subdir3/file3.txt', + 'indirect-external-link.txt => external-link.txt', + 'local-dir-link => normal-dir', + 'local-link.txt => normal-file.txt', + 'normal-dir (D)', + ' file-in-subdir.txt', + 'normal-file.txt' + ]); + test.done(); + }, + + 'External: follow only external symlinks'(test: Test) { + // GIVEN + const outdir = fs.mkdtempSync(path.join(os.tmpdir(), 'copy-tests')); + + // WHEN + copyDirectory(path.join(__dirname, 'fixtures', 'symlinks'), outdir, { + follow: FollowMode.External + }); + + // THEN + test.deepEqual(tree(outdir), [ + 'external-dir-link (D)', + ' file2.txt', + 'external-link.txt', + 'indirect-external-link.txt => external-link.txt', + 'local-dir-link => normal-dir', + 'local-link.txt => normal-file.txt', + 'normal-dir (D)', + ' file-in-subdir.txt', + 'normal-file.txt' + ]); + + test.done(); + }, + + 'exclude'(test: Test) { + // GIVEN + const outdir = fs.mkdtempSync(path.join(os.tmpdir(), 'copy-tests')); + + // WHEN + copyDirectory(path.join(__dirname, 'fixtures', 'test1'), outdir, { + exclude: [ + '*', + '!subdir2', + '!subdir2/**/*', + '.*' + ] + }); + + // THEN + test.deepEqual(tree(outdir), [ + 'subdir2 (D)', + ' empty-subdir (D)', + ' subdir3 (D)', + ' file3.txt' + ]); + test.done(); + }, +}; + +function tree(dir: string, depth = ''): string[] { + const lines = []; + for (const file of fs.readdirSync(dir).sort()) { + const filePath = path.join(dir, file); + const stat = fs.lstatSync(filePath); + if (stat.isSymbolicLink()) { + const linkDest = fs.readlinkSync(filePath); + lines.push(depth + file + ' => ' + linkDest); + } else if (stat.isDirectory()) { + lines.push(depth + file + ' (D)'); + lines.push(...tree(filePath, depth + ' ')); + } else { + lines.push(depth + file); + } + } + return lines; +} \ No newline at end of file diff --git a/packages/@aws-cdk/assets/test/fs/test.fs-fingerprint.ts b/packages/@aws-cdk/assets/test/fs/test.fs-fingerprint.ts new file mode 100644 index 0000000000000..87cf001562055 --- /dev/null +++ b/packages/@aws-cdk/assets/test/fs/test.fs-fingerprint.ts @@ -0,0 +1,108 @@ +import fs = require('fs'); +import { Test } from 'nodeunit'; +import os = require('os'); +import path = require('path'); +import { copyDirectory } from '../../lib/fs/copy'; +import { fingerprint } from '../../lib/fs/fingerprint'; + +export = { + 'single file'(test: Test) { + // GIVEN + const workdir = fs.mkdtempSync(path.join(os.tmpdir(), 'hash-tests')); + const content = 'Hello, world!'; + const input1 = path.join(workdir, 'input1.txt'); + const input2 = path.join(workdir, 'input2.txt'); + const input3 = path.join(workdir, 'input3.txt'); + fs.writeFileSync(input1, content); + fs.writeFileSync(input2, content); + fs.writeFileSync(input3, content + '.'); // add one character, hash should be different + + // WHEN + const hash1 = fingerprint(input1); + const hash2 = fingerprint(input2); + const hash3 = fingerprint(input3); + + // THEN + test.deepEqual(hash1, hash2); + test.notDeepEqual(hash3, hash1); + test.done(); + }, + + 'empty file'(test: Test) { + // GIVEN + const workdir = fs.mkdtempSync(path.join(os.tmpdir(), 'hash-tests')); + const input1 = path.join(workdir, 'empty'); + const input2 = path.join(workdir, 'empty'); + fs.writeFileSync(input1, ''); + fs.writeFileSync(input2, ''); + + // WHEN + const hash1 = fingerprint(input1); + const hash2 = fingerprint(input2); + + // THEN + test.deepEqual(hash1, hash2); + test.done(); + }, + + 'directory'(test: Test) { + // GIVEN + const srcdir = path.join(__dirname, 'fixtures', 'symlinks'); + const outdir = fs.mkdtempSync(path.join(os.tmpdir(), 'copy-tests')); + copyDirectory(srcdir, outdir); + + // WHEN + const hashSrc = fingerprint(srcdir); + const hashCopy = fingerprint(outdir); + + // THEN + test.deepEqual(hashSrc, hashCopy); + test.done(); + }, + + 'directory, rename files (fingerprint should change)'(test: Test) { + // GIVEN + const srcdir = path.join(__dirname, 'fixtures', 'symlinks'); + const cpydir = fs.mkdtempSync(path.join(os.tmpdir(), 'fingerprint-tests')); + copyDirectory(srcdir, cpydir); + + // be careful not to break a symlink + fs.renameSync(path.join(cpydir, 'normal-dir', 'file-in-subdir.txt'), path.join(cpydir, 'move-me.txt')); + + // WHEN + const hashSrc = fingerprint(srcdir); + const hashCopy = fingerprint(cpydir); + + // THEN + test.notDeepEqual(hashSrc, hashCopy); + test.done(); + }, + + 'external symlink content changes (fingerprint should change)'(test: Test) { + // GIVEN + const dir1 = fs.mkdtempSync(path.join(os.tmpdir(), 'fingerprint-tests')); + const dir2 = fs.mkdtempSync(path.join(os.tmpdir(), 'fingerprint-tests')); + const target = path.join(dir1, 'boom.txt'); + const content = 'boom'; + fs.writeFileSync(target, content); + fs.symlinkSync(target, path.join(dir2, 'link-to-boom.txt')); + + // now dir2 contains a symlink to a file in dir1 + + // WHEN + const original = fingerprint(dir2); + + // now change the contents of the target + fs.writeFileSync(target, 'changning you!'); + const afterChange = fingerprint(dir2); + + // revert the content to original and expect hash to be reverted + fs.writeFileSync(target, content); + const afterRevert = fingerprint(dir2); + + // THEN + test.notDeepEqual(original, afterChange); + test.deepEqual(afterRevert, original); + test.done(); + } +}; diff --git a/packages/@aws-cdk/assets/test/test.asset.ts b/packages/@aws-cdk/assets/test/test.asset.ts index 82147d630f893..1195c9143382c 100644 --- a/packages/@aws-cdk/assets/test/test.asset.ts +++ b/packages/@aws-cdk/assets/test/test.asset.ts @@ -1,17 +1,21 @@ import { expect, haveResource, ResourcePart, SynthUtils } from '@aws-cdk/assert'; import iam = require('@aws-cdk/aws-iam'); import cdk = require('@aws-cdk/cdk'); +import { App, Stack } from '@aws-cdk/cdk'; import cxapi = require('@aws-cdk/cx-api'); +import fs = require('fs'); import { Test } from 'nodeunit'; +import os = require('os'); import path = require('path'); import { FileAsset, ZipDirectoryAsset } from '../lib/asset'; +const SAMPLE_ASSET_DIR = path.join(__dirname, 'sample-asset-directory'); + export = { 'simple use case'(test: Test) { const stack = new cdk.Stack(); - const dirPath = path.join(__dirname, 'sample-asset-directory'); const asset = new ZipDirectoryAsset(stack, 'MyAsset', { - path: dirPath + path: SAMPLE_ASSET_DIR }); // verify that metadata contains an "aws:cdk:asset" entry with @@ -19,18 +23,17 @@ export = { const entry = asset.node.metadata.find(m => m.type === 'aws:cdk:asset'); test.ok(entry, 'found metadata entry'); - // console.error(JSON.stringify(stack.node.resolve(entry!.data))); + // verify that now the template contains parameters for this asset + const template = SynthUtils.toCloudFormation(stack); test.deepEqual(stack.node.resolve(entry!.data), { - path: dirPath, + path: SAMPLE_ASSET_DIR, id: 'MyAsset', packaging: 'zip', s3BucketParameter: 'MyAssetS3Bucket68C9B344', s3KeyParameter: 'MyAssetS3VersionKey68E1A45D', }); - // verify that now the template contains parameters for this asset - const template = SynthUtils.toCloudFormation(stack); test.equal(template.Parameters.MyAssetS3Bucket68C9B344.Type, 'String'); test.equal(template.Parameters.MyAssetS3VersionKey68E1A45D.Type, 'String'); @@ -65,6 +68,10 @@ export = { const asset = new FileAsset(stack, 'MyAsset', { path: filePath }); const entry = asset.node.metadata.find(m => m.type === 'aws:cdk:asset'); test.ok(entry, 'found metadata entry'); + + // synthesize first so "prepare" is called + const template = SynthUtils.toCloudFormation(stack); + test.deepEqual(stack.node.resolve(entry!.data), { path: filePath, packaging: 'file', @@ -74,7 +81,6 @@ export = { }); // verify that now the template contains parameters for this asset - const template = SynthUtils.toCloudFormation(stack); test.equal(template.Parameters.MyAssetS3Bucket68C9B344.Type, 'String'); test.equal(template.Parameters.MyAssetS3VersionKey68E1A45D.Type, 'String'); @@ -194,9 +200,8 @@ export = { // GIVEN const stack = new cdk.Stack(); - const location = path.join(__dirname, 'sample-asset-directory'); const resource = new cdk.CfnResource(stack, 'MyResource', { type: 'My::Resource::Type' }); - const asset = new ZipDirectoryAsset(stack, 'MyAsset', { path: location }); + const asset = new ZipDirectoryAsset(stack, 'MyAsset', { path: SAMPLE_ASSET_DIR }); // WHEN asset.addResourceMetadata(resource, 'PropName'); @@ -204,11 +209,126 @@ export = { // THEN expect(stack).notTo(haveResource('My::Resource::Type', { Metadata: { - "aws:asset:path": location, + "aws:asset:path": SAMPLE_ASSET_DIR, "aws:asset:property": "PropName" } }, ResourcePart.CompleteDefinition)); test.done(); + }, + + 'staging': { + + 'copy file assets under .assets/fingerprint.ext'(test: Test) { + const tempdir = fs.mkdtempSync(os.tmpdir()); + process.chdir(tempdir); // change current directory to somewhere in /tmp + + // GIVEN + const app = new App({ + context: { [cxapi.ASSETS_STAGING_DIR_CONTEXT]: '.assets' } + }); + const stack = new Stack(app, 'stack'); + + // WHEN + new FileAsset(stack, 'ZipFile', { + path: path.join(SAMPLE_ASSET_DIR, 'sample-zip-asset.zip') + }); + + new FileAsset(stack, 'TextFile', { + path: path.join(SAMPLE_ASSET_DIR, 'sample-asset-file.txt') + }); + + // THEN + app.run(); + test.ok(fs.existsSync(path.join(tempdir, '.assets'))); + test.ok(fs.existsSync(path.join(tempdir, '.assets', 'fdb4701ff6c99e676018ee2c24a3119b.zip'))); + fs.readdirSync(path.join(tempdir, '.assets')); + test.done(); + }, + + 'copy directory under .assets/fingerprint/**'(test: Test) { + const tempdir = fs.mkdtempSync(os.tmpdir()); + process.chdir(tempdir); // change current directory to somewhere in /tmp + + // GIVEN + const app = new App({ + context: { [cxapi.ASSETS_STAGING_DIR_CONTEXT]: '.assets' } + }); + const stack = new Stack(app, 'stack'); + + // WHEN + new ZipDirectoryAsset(stack, 'ZipDirectory', { + path: SAMPLE_ASSET_DIR + }); + + // THEN + app.run(); + test.ok(fs.existsSync(path.join(tempdir, '.assets'))); + test.ok(fs.existsSync(path.join(tempdir, '.assets', 'b550524e103eb4cf257c594fba5b9fe8', 'sample-asset-file.txt'))); + test.ok(fs.existsSync(path.join(tempdir, '.assets', 'b550524e103eb4cf257c594fba5b9fe8', 'sample-jar-asset.jar'))); + fs.readdirSync(path.join(tempdir, '.assets')); + test.done(); + }, + + 'staging path is relative if the dir is below the working directory'(test: Test) { + // GIVEN + const tempdir = fs.mkdtempSync(os.tmpdir()); + process.chdir(tempdir); // change current directory to somewhere in /tmp + + const staging = '.my-awesome-staging-directory'; + const app = new App({ + context: { + [cxapi.ASSETS_STAGING_DIR_CONTEXT]: staging, + [cxapi.ASSET_RESOURCE_METADATA_ENABLED_CONTEXT]: 'true', + } + }); + + const stack = new Stack(app, 'stack'); + + const resource = new cdk.CfnResource(stack, 'MyResource', { type: 'My::Resource::Type' }); + const asset = new ZipDirectoryAsset(stack, 'MyAsset', { path: SAMPLE_ASSET_DIR }); + + // WHEN + asset.addResourceMetadata(resource, 'PropName'); + + const session = app.run(); + const template = SynthUtils.templateForStackName(session, stack.name); + + test.deepEqual(template.Resources.MyResource.Metadata, { + "aws:asset:path": `.my-awesome-staging-directory/b550524e103eb4cf257c594fba5b9fe8`, + "aws:asset:property": "PropName" + }); + test.done(); + }, + + 'if staging directory is absolute, asset path is absolute'(test: Test) { + // GIVEN + const staging = path.resolve(fs.mkdtempSync(os.tmpdir())); + const app = new App({ + context: { + [cxapi.ASSETS_STAGING_DIR_CONTEXT]: staging, + [cxapi.ASSET_RESOURCE_METADATA_ENABLED_CONTEXT]: 'true', + } + }); + + const stack = new Stack(app, 'stack'); + + const resource = new cdk.CfnResource(stack, 'MyResource', { type: 'My::Resource::Type' }); + const asset = new ZipDirectoryAsset(stack, 'MyAsset', { path: SAMPLE_ASSET_DIR }); + + // WHEN + asset.addResourceMetadata(resource, 'PropName'); + + const session = app.run(); + const template = SynthUtils.templateForStackName(session, stack.name); + + test.deepEqual(template.Resources.MyResource.Metadata, { + "aws:asset:path": `${staging}/b550524e103eb4cf257c594fba5b9fe8`, + "aws:asset:property": "PropName" + }); + test.done(); + } + } + }; diff --git a/packages/@aws-cdk/cx-api/lib/cxapi.ts b/packages/@aws-cdk/cx-api/lib/cxapi.ts index ae76ea0ea1eac..f6ae72df744f1 100644 --- a/packages/@aws-cdk/cx-api/lib/cxapi.ts +++ b/packages/@aws-cdk/cx-api/lib/cxapi.ts @@ -177,3 +177,9 @@ export const OUTFILE_NAME = 'cdk.out'; * Disable the collection and reporting of version information. */ export const DISABLE_VERSION_REPORTING = 'aws:cdk:disable-version-reporting'; + +/** + * If this context key is set, the CDK will stage assets under the specified + * directory. Otherwise, assets will not be staged. + */ +export const ASSETS_STAGING_DIR_CONTEXT = 'aws:cdk:assets-staging-dir';