Skip to content

Commit

Permalink
Make smoke tests more deterministic (#2618)
Browse files Browse the repository at this point in the history
* sync first remote smoke tests

* update smoke test scripts
  • Loading branch information
FredKSchott authored and Nate Moore committed Mar 2, 2022
1 parent 8f08197 commit b107625
Show file tree
Hide file tree
Showing 336 changed files with 34,321 additions and 146 deletions.
34 changes: 27 additions & 7 deletions .github/workflows/nightly.yml
Original file line number Diff line number Diff line change
Expand Up @@ -60,11 +60,31 @@ jobs:
body: >
This PR is auto-generated by a nightly GitHub action.
It should automatically be merged if tests pass.
- name: Mark Pull Request for Auto-Merge
if: steps.createpr.outputs.pull-request-operation == 'created'
uses: peter-evans/enable-pull-request-automerge@v1
smoke-sync:
if: github.repository_owner == 'withastro'
runs-on: ubuntu-latest
steps:
- name: Check out code using Git
uses: actions/checkout@v2
- name: Set Node version to 16
uses: actions/setup-node@v2
with:
token: ${{ secrets.NIGHTLY_PERSONAL_GITHUB_TOKEN }}
pull-request-number: ${{ steps.createpr.outputs.pull-request-number }}
merge-method: squash
node-version: 16
cache: 'yarn'
- name: Install dependencies
run: yarn install --frozen-lockfile --ignore-engines --ignore-scripts
- name: Sync smoke tests
run: node scripts/smoke/sync.js
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Create Pull Request
id: createpr
uses: peter-evans/create-pull-request@v3
with:
token: ${{ secrets.GITHUB_TOKEN }}
commit-message: '[ci] update smoke tests (remote)'
title: '[ci] update smoke tests (remote)'
body: >
This PR is auto-generated by a nightly GitHub action.
It should automatically be merged if tests pass.
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
"examples/component/demo",
"examples/component/packages/*",
"scripts",
"scripts/smoke/*",
"smoke/*",
"packages/astro/test/fixtures/builtins/packages/*",
"packages/astro/test/fixtures/builtins-polyfillnode",
"packages/astro/test/fixtures/custom-elements/my-component-lib",
Expand Down
117 changes: 16 additions & 101 deletions scripts/smoke/index.js
Original file line number Diff line number Diff line change
@@ -1,47 +1,39 @@
/** @file Runs all smoke tests and may add extra smoke-test dependencies to `yarn.lock`. */

// @ts-check

import Zip from 'adm-zip';
import { execa } from 'execa';
import { polyfill } from '@astropub/webapi';
import { fileURLToPath } from 'node:url';
import { promises as fs } from 'node:fs';

polyfill(globalThis, { exclude: 'window document' });

/* Configuration
/* -------------------------------------------------------------------------- */

/** URL directory containing this current script. */
const scriptDir = new URL('./', import.meta.url);

/** URL directory containing the entire project. */
const rootDir = new URL('../../', import.meta.url);

/** URL directory containing the example subdirectories. */
const exampleDir = new URL('examples/', rootDir);
const smokeDir = new URL('smoke/', rootDir);

/** URL directory containing the Astro package. */
const astroDir = new URL('packages/astro/', rootDir);

/** GitHub configuration for the external "docs" Astro project. */
// const docGithubConfig = { org: 'withastro', name: 'docs', branch: 'main' };
/** Returns all child directories of the given directory. */
const getChildDirectories = async (/** @type {URL} */ dir) => {
/** @type {URL[]} */
const dirs = [];

/** GitHub configuration for the external "astro.build" Astro project. */
// const wwwGithubConfig = { org: 'withastro', name: 'astro.build', branch: 'main' };
for await (const dirent of await fs.opendir(dir)) {
if (dirent.isDirectory()) {
dirs.push(new URL(dirent.name, dir));
}
}

/* Application
/* -------------------------------------------------------------------------- */
return dirs;
};

/** Runs all smoke tests. */
async function run() {
console.log('');

const directories = await getChildDirectories(exampleDir);

// TODO Skipped the docs-main test since it is failing at the moment.
// TODO Skipped the www test since it is failing at the moment.
const directories = [...await getChildDirectories(exampleDir), ...await getChildDirectories(smokeDir)];

console.log('🤖', 'Preparing', 'yarn');

Expand All @@ -54,100 +46,23 @@ async function run() {
await execa('yarn', ['run', 'build'], { cwd: fileURLToPath(directory), stdio: 'inherit' });
} catch (err) {
console.log(err);

process.exit(1);
}

// Run with the static build too
if (directory.pathname.includes('astro.build')) {
// astro.build uses the static build, so rerunning with the flag actually negates it.
// Run with the static build too (skip for remote repos)
if (directory.pathname.includes(smokeDir.pathname)) {
continue;
}

try {
await execa('yarn', ['build', '--', '--experimental-static-build'], { cwd: fileURLToPath(directory), stdout: 'inherit', stderr: 'inherit' });
} catch (err) {
console.log(err);

process.exit(1);
}

console.log();
}
}

/* Functionality
/* -------------------------------------------------------------------------- */

/** Returns the URL to the ZIP of the given GitHub project. */
const getGithubZipURL = (/** @type {GithubOpts} */ opts) => `https://github.com/${opts.org}/${opts.name}/archive/refs/heads/${opts.branch}.zip`;

/** Returns the awaited ZIP Buffer from the given GitHub project. */
const fetchGithubZip = (/** @type {GithubOpts} */ opts) =>
fetch(getGithubZipURL(opts))
.then((response) => response.arrayBuffer())
.then((arrayBuffer) => Buffer.from(arrayBuffer));

/** Downloads a ZIP from the given GitHub project. */
const downloadGithubZip = async (/** @type {GithubOpts} */ opts) => {
/** Expected directory when the zip is downloaded. */
const githubDir = new URL(`${opts.name}-${opts.branch}`, scriptDir);

console.log('🤖', 'Downloading', `${opts.org}/${opts.name}#${opts.branch}`);

const buffer = await fetchGithubZip(opts);

console.log('🤖', 'Extracting', `${opts.org}/${opts.name}#${opts.branch}`);

new Zip(buffer).extractAllTo(fileURLToPath(scriptDir), true);

console.log('🤖', 'Preparing', `${opts.org}/${opts.name}#${opts.branch}`);

const astroPackage = await readDirectoryPackage(astroDir);

const githubPackage = await readDirectoryPackage(githubDir);

if ('astro' in Object(githubPackage.dependencies)) {
githubPackage.dependencies['astro'] = astroPackage.version;
}

if ('astro' in Object(githubPackage.devDependencies)) {
githubPackage.devDependencies['astro'] = astroPackage.version;
}

if ('astro' in Object(githubPackage.peerDependencies)) {
githubPackage.peerDependencies['astro'] = astroPackage.version;
}

await writeDirectoryPackage(githubDir, githubPackage);

return githubDir;
};

/** Returns the parsed package.json of the given directory. */
const readDirectoryPackage = async (/** @type {URL} */ dir) => JSON.parse(await fs.readFile(new URL('package.json', dir + '/'), 'utf-8'));

/** Returns upon completion of writing a package.json to the given directory. */
const writeDirectoryPackage = async (/** @type {URL} */ dir, /** @type {any} */ data) =>
await fs.writeFile(new URL('package.json', dir + '/'), JSON.stringify(data, null, ' ') + '\n');

/** Returns all child directories of the given directory. */
const getChildDirectories = async (/** @type {URL} */ dir) => {
/** @type {URL[]} */
const dirs = [];

for await (const dirent of await fs.opendir(dir)) {
if (dirent.isDirectory()) {
dirs.push(new URL(dirent.name, dir));
}
}

return dirs;
};

/* Execution
/* -------------------------------------------------------------------------- */

run();

/** @typedef {{ org: string, name: string, branch: string }} GithubOpts */
run();
97 changes: 97 additions & 0 deletions scripts/smoke/sync.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
/** @file Runs all smoke tests and may add extra smoke-test dependencies to `yarn.lock`. */

// @ts-check

import Zip from 'adm-zip';
import rimraf from 'rimraf';
import { execa } from 'execa';
import { polyfill } from '@astropub/webapi';
import { fileURLToPath } from 'node:url';
import { promises as fs } from 'node:fs';

polyfill(globalThis, { exclude: 'window document' });

/* Configuration
/* -------------------------------------------------------------------------- */

/** URL directory containing this current script. */
// const scriptDir = new URL('./', import.meta.url);

/** URL directory containing the entire project. */
const rootDir = new URL('../../', import.meta.url);

/** URL directory containing the example subdirectories. */
const exampleDir = new URL('examples/', rootDir);
const smokeDir = new URL('smoke/', rootDir);

/** URL directory containing the Astro package. */
const astroDir = new URL('packages/astro/', rootDir);

/** GitHub configuration for the external "docs" Astro project. */
const docGithubConfig = { org: 'withastro', name: 'docs', branch: 'main' };

/** GitHub configuration for the external "astro.build" Astro project. */
const wwwGithubConfig = { org: 'withastro', name: 'astro.build', branch: 'main' };

/* Application
/* -------------------------------------------------------------------------- */

/** Runs all smoke tests. */
async function run() {
await downloadGithubZip(docGithubConfig);
await downloadGithubZip(wwwGithubConfig);
await execa('yarn', [], { cwd: fileURLToPath(rootDir), stdout: 'inherit', stderr: 'inherit' });
}

/* Functionality
/* -------------------------------------------------------------------------- */

/** Returns the URL to the ZIP of the given GitHub project. */
const getGithubZipURL = (/** @type {GithubOpts} */ opts) => `https://github.com/${opts.org}/${opts.name}/archive/refs/heads/${opts.branch}.zip`;

/** Returns the awaited ZIP Buffer from the given GitHub project. */
const fetchGithubZip = (/** @type {GithubOpts} */ opts) =>
fetch(getGithubZipURL(opts))
.then((response) => response.arrayBuffer())
.then((arrayBuffer) => Buffer.from(arrayBuffer));

/** Downloads a ZIP from the given GitHub project. */
const downloadGithubZip = async (/** @type {GithubOpts} */ opts) => {
/** Expected directory when the zip is downloaded. */
const githubDir = new URL(`${opts.name}-${opts.branch}`, smokeDir);
/** Whether the expected directory is already available */
rimraf.sync(fileURLToPath(githubDir));
console.log('🤖', 'Downloading', `${opts.org}/${opts.name}#${opts.branch}`);
const buffer = await fetchGithubZip(opts);
console.log('🤖', 'Extracting', `${opts.org}/${opts.name}#${opts.branch}`);
new Zip(buffer).extractAllTo(fileURLToPath(smokeDir), true);
console.log('🤖', 'Preparing', `${opts.org}/${opts.name}#${opts.branch}`);
const astroPackage = await readDirectoryPackage(astroDir);
const githubPackage = await readDirectoryPackage(githubDir);
if ('astro' in Object(githubPackage.dependencies)) {
githubPackage.dependencies['astro'] = astroPackage.version;
}
if ('astro' in Object(githubPackage.devDependencies)) {
githubPackage.devDependencies['astro'] = astroPackage.version;
}
if ('astro' in Object(githubPackage.peerDependencies)) {
githubPackage.peerDependencies['astro'] = astroPackage.version;
}
await writeDirectoryPackage(githubDir, githubPackage);
rimraf.sync(fileURLToPath(new URL(`yarn.lock`, githubDir)));
rimraf.sync(fileURLToPath(new URL(`package-lock.json`, githubDir)));
};

/** Returns the parsed package.json of the given directory. */
const readDirectoryPackage = async (/** @type {URL} */ dir) => JSON.parse(await fs.readFile(new URL('package.json', dir + '/'), 'utf-8'));

/** Returns upon completion of writing a package.json to the given directory. */
const writeDirectoryPackage = async (/** @type {URL} */ dir, /** @type {any} */ data) =>
await fs.writeFile(new URL('package.json', dir + '/'), JSON.stringify(data, null, ' ') + '\n');

/* Execution
/* -------------------------------------------------------------------------- */

run();

/** @typedef {{ org: string, name: string, branch: string }} GithubOpts */
Loading

0 comments on commit b107625

Please sign in to comment.