chore(cli): prevent test interference

Our CLI unit tests were interfering with each other because they
were writing files from and to the current directory, which is
shared between all of them.

Solve it by making a non-writeable directory before running the
tests, so that the tests that do that start throwing errors and
we can identify them. Then fix those.

packages/aws-cdk/jest.config.js

@@ -20,6 +20,5 @@ module.exports = {
 
     // We have many tests here that commonly time out
     testTimeout: 30_000,
-    // These tests are too chatty. Shush.
-    silent: true,
+    setupFilesAfterEnv: ["<rootDir>/test/jest-setup-after-env.ts"],
 };

packages/aws-cdk/test/diff.test.ts

@@ -1,4 +1,6 @@
 import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
 import { Writable } from 'stream';
 import { StringDecoder } from 'string_decoder';
 import * as cxschema from '@aws-cdk/cloud-assembly-schema';
@@ -12,6 +14,22 @@ import { CdkToolkit } from '../lib/cdk-toolkit';
 let cloudExecutable: MockCloudExecutable;
 let cloudFormation: jest.Mocked<Deployments>;
 let toolkit: CdkToolkit;
+let oldDir: string;
+let tmpDir: string;
+
+beforeAll(() => {
+  // The toolkit writes and checks for temporary files in the current directory,
+  // so run these tests in a tempdir so they don't interfere with each other
+  // and other tests.
+  oldDir = process.cwd();
+  tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'aws-cdk-test'));
+  process.chdir(tmpDir);
+});
+
+afterAll(() => {
+  process.chdir(oldDir);
+  fs.rmSync(tmpDir, { recursive: true, force: true });
+});
 
 describe('fixed template', () => {
   const templatePath = 'oldTemplate.json';

packages/aws-cdk/test/jest-setup-after-env.ts

@@ -0,0 +1,64 @@
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+
+/**
+ * Global test setup for Jest tests
+ *
+ * It's easy to accidentally write tests that interfere with each other by
+ * writing files to disk in the "current directory". To prevent this, the global
+ * test setup creates a directory in the temporary directory and chmods it to
+ * being non-writable. That way, whenever a test tries to write to the current
+ * directory, it will produce an error and we'll be able to find and fix the
+ * test.
+ *
+ * If you see `EACCES: permission denied`, you have a test that creates files
+ * in the current directory, and you should be sure to do it in a temporary
+ * directory that you clean up afterwards.
+ *
+ * ## Alternate approach
+ *
+ * I tried an approach where I would automatically try to create and clean up
+ * temp directories for every test, but it was introducing too many conflicts
+ * with existing test behavior (around specific ordering of temp directory
+ * creation and cleanup tasks that are already present) in many places that I
+ * didn't want to go and chase down.
+ *
+ */
+
+let tmpDir: string;
+let oldDir: string;
+
+beforeAll(() => {
+  tmpDir = path.join(os.tmpdir(), 'cdk-nonwritable');
+  if (!fs.existsSync(tmpDir)) {
+    fs.mkdirSync(tmpDir);
+    fs.chmodSync(tmpDir, 0o500);
+  }
+  oldDir = process.cwd();
+  process.chdir(tmpDir);
+  tmpDir = process.cwd(); // This will have resolved symlinks
+});
+
+/**
+ * We need a cleanup here
+ *
+ * 99% of the time, Jest runs the tests in a subprocess and this isn't
+ * necessary because we would have `chdir`ed in the subprocess.
+ *
+ * But sometimes we ask Jest with `-i` to run the tests in the main process,
+ * or if you only ask for a single test suite Jest runs the tests in the main
+ * process, and then we `chdir`ed the main process away.
+ *
+ * Jest will then try to write the `coverage` directory to the readonly directory,
+ * and fail. Chdir back to the original dir.
+ *
+ * Only if we are still in the tempdir, because if not then some other temporary
+ * directory cleanup block has already done the same and we shouldn't interfere
+ * with that.
+ */
+afterAll(() => {
+  if (process.cwd() === tmpDir) {
+    process.chdir(oldDir);
+  }
+});

packages/aws-cdk/test/notices.test.ts

@@ -455,23 +455,20 @@ describe(CachedDataSource, () => {
   });
 
   test('retrieves data from the delegate when the file cannot be read', async () => {
-    const debugSpy = jest.spyOn(logging, 'debug');
+    const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'cdk-test'));
+    try {
+      const debugSpy = jest.spyOn(logging, 'debug');
 
-    if (fs.existsSync('does-not-exist.json')) {
-      fs.unlinkSync('does-not-exist.json');
-    }
-
-    const dataSource = dataSourceWithDelegateReturning(freshData, 'does-not-exist.json');
+      const dataSource = dataSourceWithDelegateReturning(freshData, `${tmpDir}/does-not-exist.json`);
 
-    const notices = await dataSource.fetch();
-
-    expect(notices).toEqual(freshData);
-    expect(debugSpy).not.toHaveBeenCalled();
+      const notices = await dataSource.fetch();
 
-    debugSpy.mockRestore();
+      expect(notices).toEqual(freshData);
+      expect(debugSpy).not.toHaveBeenCalled();
 
-    if (fs.existsSync('does-not-exist.json')) {
-      fs.unlinkSync('does-not-exist.json');
+      debugSpy.mockRestore();
+    } finally {
+      fs.rmSync(tmpDir, { recursive: true, force: true });
     }
   });