chore: fix CLI integration tests (#4198)

* chore: fix CLI integration tests

Fix breakage introduced by #3782, which changed the wrong string.

Update other expectation to match change introduced in #3772.

* Use substitutions in error message

* Rewrite how CLI tests are being run

* Extract environment setup from tests

* Need to not ignore integ test Javascript files

packages/@aws-cdk/aws-codestar/.npmignore

@@ -16,4 +16,7 @@ dist
 .LAST_PACKAGE
 .jsii
 
-*.tsbuildinfo
+*.tsbuildinfo
+
+# Include .jsii
+!.jsii

packages/@aws-cdk/aws-lakeformation/.npmignore

@@ -16,4 +16,7 @@ dist
 .LAST_PACKAGE
 .jsii
 
-*.tsbuildinfo
+*.tsbuildinfo
+
+# Include .jsii
+!.jsii

packages/@aws-cdk/aws-managedblockchain/.npmignore

@@ -16,4 +16,7 @@ dist
 .LAST_PACKAGE
 .jsii
 
-*.tsbuildinfo
+*.tsbuildinfo
+
+# Include .jsii
+!.jsii

packages/@aws-cdk/aws-qldb/.npmignore

@@ -16,4 +16,7 @@ dist
 .LAST_PACKAGE
 .jsii
 
-*.tsbuildinfo
+*.tsbuildinfo
+
+# Include .jsii
+!.jsii

packages/@aws-cdk/cloudformation-diff/lib/format.ts

@@ -66,7 +66,7 @@ function formatSecurityChangesWithBanner(formatter: Formatter, templateDiff: Tem
   formatter.formatIamChanges(templateDiff.iamChanges);
   formatter.formatSecurityGroupChanges(templateDiff.securityGroupChanges);
 
-  formatter.warning(`(NOTE: There may be security-related changes not in this list. See http://bit.ly/cdk-2EhF7Np)`);
+  formatter.warning(`(NOTE: There may be security-related changes not in this list. See https://github.com/aws/aws-cdk/issues/1299)`);
   formatter.printSectionFooter();
 }
 

packages/aws-cdk/.npmignore

@@ -12,5 +12,6 @@ dist
 *.snk
 
 !lib/init-templates/*/*/tsconfig.json
+!test/integ/cli/**/*.js
 
 *.tsbuildinfo

packages/aws-cdk/package.json

@@ -16,8 +16,8 @@
     "package": "cdk-package",
     "build+test+package": "npm run build+test && npm run package",
     "build+test": "npm run build && npm test",
-    "integ-cli": "CDK_REPO=$PWD/../.. test/integ/cli/test.sh",
-    "integ-init": "CDK_DIST=$PWD/../../dist test/integ/init/test-all.sh"
+    "integ-cli": "test/integ/run-against-repo test/integ/cli/test.sh",
+    "integ-init": "test/integ/run-against-dist test/integ/init/test-all.sh"
   },
   "cdk-build": {
     "pre": [

packages/aws-cdk/test/integ/cli/README.md

@@ -0,0 +1,25 @@
+# CDK CLI Integration Tests
+
+These tests require AWS credentials, and exercise various aspects of the
+CLI on a simple JavaScript CDK app (stored in `app/`).
+
+## Entry point
+
+```
+./test.sh
+```
+
+## Configuration
+
+AWS credentials must be configured.
+
+Optional configuration:
+
+* `AWS_DEFAULT_REGION`, what region to deploy the stacks in.
+* `STACK_NAME_PREFIX`, used to run multiple instances of these tests in the
+  same account side-by-side without them stepping on each other. Using
+  a unique name on every run is risky since the account may overflow with
+  stacks if cleanup happens to fail. Defaults based on the value of `IS_CANARY`
+  if not supplied.
+* `IS_CANARY`, true or false. Affects the default stack name prefix to make
+  integration test and canary runs unique.

packages/aws-cdk/test/integ/cli/common.bash

@@ -1,39 +1,36 @@
 set -eu
 scriptdir=$(cd $(dirname $0) && pwd)
-source $scriptdir/../common/util.bash
-
-CDK_REPO="${CDK_REPO:-}"
-
-# If CDK_REPO is set to point to the root of the CDK source repository
-# the CLI and modules will be taken from there. Otherwise, we will honor
-# the usual IS_CANARY flag for integration test-vs-canary mode switch.
-if [ -z "${CDK_REPO}" -a -z "${TESTS_PREPARED:-}" ]; then
-    prepare_toolkit
-    preload_npm_packages
-
-    # Only do this once
-    export TESTS_PREPARED=1
-fi
-
-if [[ "${CDK_REPO}" != "" ]]; then
-    export CDK_REPO=$(cd $CDK_REPO && pwd)
-    alias cdk=$CDK_REPO/packages/aws-cdk/bin/cdk
-fi
+cd ${scriptdir}
 
 if [[ -z "${CREDS_SET:-}" ]]; then
     # Check that credentials are configured
     aws sts get-caller-identity > /dev/null
     export CREDS_SET=1
 fi
 
-cd ${scriptdir}
 
-if ${IS_CANARY:-false}; then
-  export STACK_NAME_PREFIX=cdk-toolkit-canary
-else
-  export STACK_NAME_PREFIX=cdk-toolkit-integration
+if [[ "${STACK_NAME_PREFIX:-}" == "" ]]; then
+  if ${IS_CANARY:-false}; then
+    export STACK_NAME_PREFIX=cdk-toolkit-canary
+  else
+    export STACK_NAME_PREFIX=cdk-toolkit-integration
+  fi
 fi
 
+#----------------------------------------------------------------------------------
+# Only functions from here on out
+
+function log() {
+  echo >&2 "| $@"
+}
+
+function header() {
+  log
+  log "============================================================================================"
+  log $@
+  log "============================================================================================"
+}
+
 function cleanup_stack() {
   local stack_arn=$1
   echo "| ensuring ${stack_arn} is cleaned up"
@@ -42,50 +39,47 @@ function cleanup_stack() {
   fi
 }
 
+integ_test_dir=/tmp/cdk-integ-test
+
+# Prepare the app fixture
+#
+# If this is done in the main test script, it will be skipped
+# in the subprocess scripts since the app fixture can just be reused.
+function prepare_fixture() {
+  if [[ -z "${FIXTURE_PREPARED:-}" ]]; then
+    log "Preparing app fixture..."
+
+    rm -rf $integ_test_dir
+    mkdir -p $integ_test_dir
+    cp -R app/* $integ_test_dir
+    cd $integ_test_dir
+
+    npm install \
+        @aws-cdk/core \
+        @aws-cdk/aws-sns \
+        @aws-cdk/aws-iam \
+        @aws-cdk/aws-lambda \
+        @aws-cdk/aws-ssm \
+        @aws-cdk/aws-ecr-assets \
+        @aws-cdk/aws-ec2
+
+    echo "| setup complete at: $PWD"
+    echo "| 'cdk' is: $(type -p cdk)"
+
+    export FIXTURE_PREPARED=1
+  fi
+}
+
 function cleanup() {
   cleanup_stack ${STACK_NAME_PREFIX}-test-1
   cleanup_stack ${STACK_NAME_PREFIX}-test-2
   cleanup_stack ${STACK_NAME_PREFIX}-iam-test
 }
 
-function install_dep() {
-  local dep=$1
-
-  if [ -n "${CDK_REPO}" ]; then
-    mkdir -p node_modules/@aws-cdk
-    local source="${CDK_REPO}/packages/${dep}"
-    local target="$PWD/node_modules/${dep}"
-    echo "| symlinking dependency ${target} => ${source}"
-    ln -s ${source} ${target}
-  else
-    echo "| installing dependency ${dep}"
-    npm i --no-save ${dep}
-  fi
-}
-
 function setup() {
   cleanup
-  rm -rf /tmp/cdk-integ-test
-  mkdir -p /tmp/cdk-integ-test
-  cp -R app/* /tmp/cdk-integ-test
-  cd /tmp/cdk-integ-test
-
-  if [ -n "${CDK_REPO}" ]; then
-    local cdk_bin="${CDK_REPO}/packages/aws-cdk/bin"
-    echo "| adding ${cdk_bin} to PATH"
-    export PATH=${cdk_bin}:$PATH
-  fi
-
-  install_dep @aws-cdk/core
-  install_dep @aws-cdk/aws-sns
-  install_dep @aws-cdk/aws-iam
-  install_dep @aws-cdk/aws-lambda
-  install_dep @aws-cdk/aws-ssm
-  install_dep @aws-cdk/aws-ecr-assets
-  install_dep @aws-cdk/aws-ec2
-
-  echo "| setup complete at: $PWD"
-  echo "| 'cdk' is: $(which cdk)"
+  prepare_fixture
+  cd $integ_test_dir
 }
 
 function fail() {

packages/aws-cdk/test/integ/cli/test-cdk-context.sh

@@ -4,10 +4,6 @@ scriptdir=$(cd $(dirname $0) && pwd)
 source ${scriptdir}/common.bash
 # ----------------------------------------------------------
 
-rm -rf /tmp/cdk-integ-test
-mkdir -p /tmp/cdk-integ-test
-cd /tmp/cdk-integ-test
-
 cat > cdk.context.json <<HERE
 {
   "contextkey": "this is the context value"
@@ -25,4 +21,6 @@ cdk context 2>&1 | grep "this is the context value" > /dev/null && { echo "Shoul
 # Test that forced delete of the context key does not error
 cdk context -f --reset contextkey
 
+rm -f cdk.context.json
+
 echo "✅  success"

packages/aws-cdk/test/integ/cli/test-cdk-ssm-parameter-provider-error.sh

@@ -15,7 +15,8 @@ function cdk_synth() {
 }
 
 assert "cdk_synth ${STACK_NAME_PREFIX}-missing-ssm-parameter -c test:ssm-parameter-name=${parameterName}" <<HERE
-SSM parameter not available in account ${account}, region ${region}: ${parameterName}
+[Error at /${STACK_NAME_PREFIX}-missing-ssm-parameter] SSM parameter not available in account ${account}, region ${region}: /does/not/exist
+Found errors
 HERE
 
 echo "✅  success"

packages/aws-cdk/test/integ/cli/test.sh

@@ -5,6 +5,8 @@ source ${scriptdir}/common.bash
 
 header CLI Integration Tests
 
+prepare_fixture
+
 for test in $(cd ${scriptdir} && ls test-*.sh); do
   echo "============================================================================================"
   echo "${test}"