Make smoke tests more deterministic

.github/workflows/nightly.yml

@@ -60,11 +60,31 @@ jobs:
           body: >
             This PR is auto-generated by a nightly GitHub action. 
             It should automatically be merged if tests pass.
-
-      - name: Mark Pull Request for Auto-Merge
-        if: steps.createpr.outputs.pull-request-operation == 'created'
-        uses: peter-evans/enable-pull-request-automerge@v1
+
+  smoke-sync:
+    if: github.repository_owner == 'withastro'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code using Git
+        uses: actions/checkout@v2
+      - name: Set Node version to 16
+        uses: actions/setup-node@v2
         with:
-          token: ${{ secrets.NIGHTLY_PERSONAL_GITHUB_TOKEN }}
-          pull-request-number: ${{ steps.createpr.outputs.pull-request-number }}
-          merge-method: squash
+          node-version: 16
+          cache: 'yarn'
+      - name: Install dependencies
+        run: yarn install --frozen-lockfile --ignore-engines --ignore-scripts
+      - name: Sync smoke tests
+        run: node scripts/smoke/sync.js
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Create Pull Request
+        id: createpr
+        uses: peter-evans/create-pull-request@v3
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          commit-message: '[ci] update smoke tests (remote)'
+          title: '[ci] update smoke tests (remote)'
+          body: >
+            This PR is auto-generated by a nightly GitHub action. 
+            It should automatically be merged if tests pass.

package.json

@@ -28,7 +28,7 @@
     "examples/component/demo",
     "examples/component/packages/*",
     "scripts",
-    "scripts/smoke/*",
+    "smoke/*",
     "packages/astro/test/fixtures/builtins/packages/*",
     "packages/astro/test/fixtures/builtins-polyfillnode",
     "packages/astro/test/fixtures/custom-elements/my-component-lib",

scripts/smoke/index.js

@@ -1,47 +1,39 @@
 /** @file Runs all smoke tests and may add extra smoke-test dependencies to `yarn.lock`. */
-
 // @ts-check
 
-import Zip from 'adm-zip';
 import { execa } from 'execa';
 import { polyfill } from '@astropub/webapi';
 import { fileURLToPath } from 'node:url';
 import { promises as fs } from 'node:fs';
 
 polyfill(globalThis, { exclude: 'window document' });
 
-/* Configuration
-/* -------------------------------------------------------------------------- */
-
-/** URL directory containing this current script. */
-const scriptDir = new URL('./', import.meta.url);
-
 /** URL directory containing the entire project. */
 const rootDir = new URL('../../', import.meta.url);
 
 /** URL directory containing the example subdirectories. */
 const exampleDir = new URL('examples/', rootDir);
+const smokeDir = new URL('smoke/', rootDir);
 
-/** URL directory containing the Astro package. */
-const astroDir = new URL('packages/astro/', rootDir);
-
-/** GitHub configuration for the external "docs" Astro project. */
-// const docGithubConfig = { org: 'withastro', name: 'docs', branch: 'main' };
+/** Returns all child directories of the given directory. */
+const getChildDirectories = async (/** @type {URL} */ dir) => {
+	/** @type {URL[]} */
+	const dirs = [];
 
-/** GitHub configuration for the external "astro.build" Astro project. */
-// const wwwGithubConfig = { org: 'withastro', name: 'astro.build', branch: 'main' };
+	for await (const dirent of await fs.opendir(dir)) {
+		if (dirent.isDirectory()) {
+			dirs.push(new URL(dirent.name, dir));
+		}
+	}
 
-/* Application
-/* -------------------------------------------------------------------------- */
+	return dirs;
+};
 
 /** Runs all smoke tests. */
 async function run() {
 	console.log('');
 
-	const directories = await getChildDirectories(exampleDir);
-
-	// TODO Skipped the docs-main test since it is failing at the moment.
-	// TODO Skipped the www test since it is failing at the moment.
+	const directories = [...await getChildDirectories(exampleDir), ...await getChildDirectories(smokeDir)];
 
 	console.log('🤖', 'Preparing', 'yarn');
 
@@ -54,100 +46,23 @@ async function run() {
 			await execa('yarn', ['run', 'build'], { cwd: fileURLToPath(directory), stdio: 'inherit' });
 		} catch (err) {
 			console.log(err);
-
 			process.exit(1);
 		}
 
-		// Run with the static build too
-		if (directory.pathname.includes('astro.build')) {
-			// astro.build uses the static build, so rerunning with the flag actually negates it.
+		// Run with the static build too (skip for remote repos)
+		if (directory.pathname.includes(smokeDir.pathname)) {
 			continue;
 		}
 
 		try {
 			await execa('yarn', ['build', '--', '--experimental-static-build'], { cwd: fileURLToPath(directory), stdout: 'inherit', stderr: 'inherit' });
 		} catch (err) {
 			console.log(err);
-
 			process.exit(1);
 		}
 
 		console.log();
 	}
 }
 
-/* Functionality
-/* -------------------------------------------------------------------------- */
-
-/** Returns the URL to the ZIP of the given GitHub project. */
-const getGithubZipURL = (/** @type {GithubOpts} */ opts) => `https://github.com/${opts.org}/${opts.name}/archive/refs/heads/${opts.branch}.zip`;
-
-/** Returns the awaited ZIP Buffer from the given GitHub project. */
-const fetchGithubZip = (/** @type {GithubOpts} */ opts) =>
-	fetch(getGithubZipURL(opts))
-		.then((response) => response.arrayBuffer())
-		.then((arrayBuffer) => Buffer.from(arrayBuffer));
-
-/** Downloads a ZIP from the given GitHub project. */
-const downloadGithubZip = async (/** @type {GithubOpts} */ opts) => {
-	/** Expected directory when the zip is downloaded. */
-	const githubDir = new URL(`${opts.name}-${opts.branch}`, scriptDir);
-
-	console.log('🤖', 'Downloading', `${opts.org}/${opts.name}#${opts.branch}`);
-
-	const buffer = await fetchGithubZip(opts);
-
-	console.log('🤖', 'Extracting', `${opts.org}/${opts.name}#${opts.branch}`);
-
-	new Zip(buffer).extractAllTo(fileURLToPath(scriptDir), true);
-
-	console.log('🤖', 'Preparing', `${opts.org}/${opts.name}#${opts.branch}`);
-
-	const astroPackage = await readDirectoryPackage(astroDir);
-
-	const githubPackage = await readDirectoryPackage(githubDir);
-
-	if ('astro' in Object(githubPackage.dependencies)) {
-		githubPackage.dependencies['astro'] = astroPackage.version;
-	}
-
-	if ('astro' in Object(githubPackage.devDependencies)) {
-		githubPackage.devDependencies['astro'] = astroPackage.version;
-	}
-
-	if ('astro' in Object(githubPackage.peerDependencies)) {
-		githubPackage.peerDependencies['astro'] = astroPackage.version;
-	}
-
-	await writeDirectoryPackage(githubDir, githubPackage);
-
-	return githubDir;
-};
-
-/** Returns the parsed package.json of the given directory. */
-const readDirectoryPackage = async (/** @type {URL} */ dir) => JSON.parse(await fs.readFile(new URL('package.json', dir + '/'), 'utf-8'));
-
-/** Returns upon completion of writing a package.json to the given directory. */
-const writeDirectoryPackage = async (/** @type {URL} */ dir, /** @type {any} */ data) =>
-	await fs.writeFile(new URL('package.json', dir + '/'), JSON.stringify(data, null, '  ') + '\n');
-
-/** Returns all child directories of the given directory. */
-const getChildDirectories = async (/** @type {URL} */ dir) => {
-	/** @type {URL[]} */
-	const dirs = [];
-
-	for await (const dirent of await fs.opendir(dir)) {
-		if (dirent.isDirectory()) {
-			dirs.push(new URL(dirent.name, dir));
-		}
-	}
-
-	return dirs;
-};
-
-/* Execution
-/* -------------------------------------------------------------------------- */
-
-run();
-
-/** @typedef {{ org: string, name: string, branch: string }} GithubOpts */
+run();

scripts/smoke/sync.js

@@ -0,0 +1,97 @@
+/** @file Runs all smoke tests and may add extra smoke-test dependencies to `yarn.lock`. */
+
+// @ts-check
+
+import Zip from 'adm-zip';
+import rimraf from 'rimraf';
+import { execa } from 'execa';
+import { polyfill } from '@astropub/webapi';
+import { fileURLToPath } from 'node:url';
+import { promises as fs } from 'node:fs';
+
+polyfill(globalThis, { exclude: 'window document' });
+
+/* Configuration
+/* -------------------------------------------------------------------------- */
+
+/** URL directory containing this current script. */
+// const scriptDir = new URL('./', import.meta.url);
+
+/** URL directory containing the entire project. */
+const rootDir = new URL('../../', import.meta.url);
+
+/** URL directory containing the example subdirectories. */
+const exampleDir = new URL('examples/', rootDir);
+const smokeDir = new URL('smoke/', rootDir);
+
+/** URL directory containing the Astro package. */
+const astroDir = new URL('packages/astro/', rootDir);
+
+/** GitHub configuration for the external "docs" Astro project. */
+const docGithubConfig = { org: 'withastro', name: 'docs', branch: 'main' };
+
+/** GitHub configuration for the external "astro.build" Astro project. */
+const wwwGithubConfig = { org: 'withastro', name: 'astro.build', branch: 'main' };
+
+/* Application
+/* -------------------------------------------------------------------------- */
+
+/** Runs all smoke tests. */
+async function run() {
+	await downloadGithubZip(docGithubConfig);
+  await downloadGithubZip(wwwGithubConfig);
+	await execa('yarn', [], { cwd: fileURLToPath(rootDir), stdout: 'inherit', stderr: 'inherit' });
+}
+
+/* Functionality
+/* -------------------------------------------------------------------------- */
+
+/** Returns the URL to the ZIP of the given GitHub project. */
+const getGithubZipURL = (/** @type {GithubOpts} */ opts) => `https://github.com/${opts.org}/${opts.name}/archive/refs/heads/${opts.branch}.zip`;
+
+/** Returns the awaited ZIP Buffer from the given GitHub project. */
+const fetchGithubZip = (/** @type {GithubOpts} */ opts) =>
+	fetch(getGithubZipURL(opts))
+		.then((response) => response.arrayBuffer())
+		.then((arrayBuffer) => Buffer.from(arrayBuffer));
+
+/** Downloads a ZIP from the given GitHub project. */
+const downloadGithubZip = async (/** @type {GithubOpts} */ opts) => {
+	/** Expected directory when the zip is downloaded. */
+	const githubDir = new URL(`${opts.name}-${opts.branch}`, smokeDir);
+	/** Whether the expected directory is already available */
+  rimraf.sync(fileURLToPath(githubDir));
+  console.log('🤖', 'Downloading', `${opts.org}/${opts.name}#${opts.branch}`);
+  const buffer = await fetchGithubZip(opts);
+  console.log('🤖', 'Extracting', `${opts.org}/${opts.name}#${opts.branch}`);
+  new Zip(buffer).extractAllTo(fileURLToPath(smokeDir), true);
+  console.log('🤖', 'Preparing', `${opts.org}/${opts.name}#${opts.branch}`);
+  const astroPackage = await readDirectoryPackage(astroDir);
+  const githubPackage = await readDirectoryPackage(githubDir);
+  if ('astro' in Object(githubPackage.dependencies)) {
+    githubPackage.dependencies['astro'] = astroPackage.version;
+  }
+  if ('astro' in Object(githubPackage.devDependencies)) {
+    githubPackage.devDependencies['astro'] = astroPackage.version;
+  }
+  if ('astro' in Object(githubPackage.peerDependencies)) {
+    githubPackage.peerDependencies['astro'] = astroPackage.version;
+  }
+  await writeDirectoryPackage(githubDir, githubPackage);
+  rimraf.sync(fileURLToPath(new URL(`yarn.lock`, githubDir)));
+  rimraf.sync(fileURLToPath(new URL(`package-lock.json`, githubDir)));
+};
+
+/** Returns the parsed package.json of the given directory. */
+const readDirectoryPackage = async (/** @type {URL} */ dir) => JSON.parse(await fs.readFile(new URL('package.json', dir + '/'), 'utf-8'));
+
+/** Returns upon completion of writing a package.json to the given directory. */
+const writeDirectoryPackage = async (/** @type {URL} */ dir, /** @type {any} */ data) =>
+	await fs.writeFile(new URL('package.json', dir + '/'), JSON.stringify(data, null, '  ') + '\n');
+
+/* Execution
+/* -------------------------------------------------------------------------- */
+
+run();
+
+/** @typedef {{ org: string, name: string, branch: string }} GithubOpts */