Skip to content

Commit

Permalink
feat(e2e): smoketest macOS .dmg files COMPASS-8709 (mongodb-js#6582)
Browse files Browse the repository at this point in the history
* run smoketests on mac

* promisified async spawn helper rather

* the app might already exist

* try and preserve permissions

* print args

* we seem to need that

* basic test

* all the webdriver logs

* really enable chromedriver's verbose logging

* MOAR log output

* remove settings dir before starting

* comment for clarification

* run this on the GUI machines

* ignore hadron-build-info.json

* comments
  • Loading branch information
lerouxb authored Dec 27, 2024
1 parent 1058ad5 commit 6bf3d97
Show file tree
Hide file tree
Showing 10 changed files with 218 additions and 30 deletions.
24 changes: 12 additions & 12 deletions .evergreen/buildvariants-and-tasks.in.yml
Original file line number Diff line number Diff line change
Expand Up @@ -80,18 +80,18 @@ const SMOKETEST_BUILD_VARIANTS = [
// run_on: 'rhel80-large',
// depends_on: 'package-rhel',
// },
// {
// name: 'smoketest-macos-x64',
// display_name: 'Smoketest MacOS Intel',
// run_on: 'macos-14',
// depends_on: 'package-macos-x64',
// },
// {
// name: 'smoketest-macos-arm',
// display_name: 'Smoketest MacOS Arm64',
// run_on: 'macos-14-arm64',
// depends_on: 'package-macos-arm',
// }
{
name: 'smoketest-macos-x64',
display_name: 'Smoketest MacOS Intel',
run_on: 'macos-14-gui',
depends_on: 'package-macos-x64',
},
{
name: 'smoketest-macos-arm',
display_name: 'Smoketest MacOS Arm64',
run_on: 'macos-14-arm64-gui',
depends_on: 'package-macos-arm',
}
];
const TEST_PACKAGED_APP_BUILD_VARIANTS = [
Expand Down
16 changes: 16 additions & 0 deletions .evergreen/buildvariants-and-tasks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,22 @@ buildvariants:
variant: package-ubuntu
tasks:
- name: smoketest-compass
- name: smoketest-macos-x64-compass
display_name: Smoketest MacOS Intel (compass)
run_on: macos-14-gui
depends_on:
- name: package-compass
variant: package-macos-x64
tasks:
- name: smoketest-compass
- name: smoketest-macos-arm-compass
display_name: Smoketest MacOS Arm64 (compass)
run_on: macos-14-arm64-gui
depends_on:
- name: package-compass
variant: package-macos-arm
tasks:
- name: smoketest-compass
- name: test-eol-servers
display_name: Test EoL Servers
run_on: ubuntu1804-large
Expand Down
2 changes: 1 addition & 1 deletion .evergreen/functions.yml
Original file line number Diff line number Diff line change
Expand Up @@ -674,7 +674,7 @@ functions:
export COMPASS_E2E_DISABLE_CLIPBOARD_USAGE="true"
fi
npm run --workspace compass-e2e-tests smoketest
npm run --unsafe-perm --workspace compass-e2e-tests smoketest
test-web-sandbox:
- command: shell.exec
Expand Down
1 change: 1 addition & 0 deletions packages/compass-e2e-tests/.prettierignore
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,4 @@
fixtures
.nyc_output
coverage
hadron-build-info.json
1 change: 1 addition & 0 deletions packages/compass-e2e-tests/helpers/chrome-startup-flags.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
// Copied from https://github.com/webdriverio/webdriverio/blob/1825c633aead82bc650dff1f403ac30cff7c7cb3/packages/devtools/src/constants.ts
// These are the default flags that webdriverio uses to start Chrome driver.
// NOTE: this has since been removed along with the devtools automation protocol https://github.com/webdriverio/webdriverio/commit/28e64e439ffc36a95f24aeda9f1d21111429dfa3#diff-6ea151d6c0687197931735239f397b7f5f0140a588c5b2b82ff584bbe73be069
const DEFAULT_WEBDRIVER_FLAGS = [
// suppresses Save Password prompt window
'--enable-automation',
Expand Down
21 changes: 15 additions & 6 deletions packages/compass-e2e-tests/helpers/compass.ts
Original file line number Diff line number Diff line change
Expand Up @@ -547,7 +547,7 @@ async function processCommonOpts({

// https://webdriver.io/docs/options/#webdriver-options
const webdriverOptions = {
logLevel: 'warn' as const, // info is super verbose right now
logLevel: 'trace' as const,
outputDir: webdriverLogPath,
};

Expand Down Expand Up @@ -601,7 +601,9 @@ async function startCompassElectron(
// See https://www.electronjs.org/docs/latest/api/command-line-switches#--enable-loggingfile
'--enable-logging=file',
// See https://www.electronjs.org/docs/latest/api/command-line-switches#--log-filepath
`--log-file=${electronLogFile}`
`--log-file=${electronLogFile}`,
// See https://chromium.googlesource.com/chromium/src/+/master/docs/chrome_os_logging.md
'--log-level=0'
);

if (opts.extraSpawnArgs) {
Expand Down Expand Up @@ -643,9 +645,13 @@ async function startCompassElectron(
},
// from https://github.com/webdriverio-community/wdio-electron-service/blob/32457f60382cb4970c37c7f0a19f2907aaa32443/packages/wdio-electron-service/src/launcher.ts#L102
'wdio:enforceWebDriverClassic': true,
},
'wdio:chromedriverOptions': {
// TODO: enable logging so we don't have to debug things blindly
'wdio:chromedriverOptions': {
// enable logging so we don't have to debug things blindly
// This goes in .log/webdriver/wdio-chromedriver-*.log. It is the
// chromedriver log and since this is verbose it also contains the
// stdout of the electron main process.
verbose: true,
},
},
...webdriverOptions,
...wdioOptions,
Expand All @@ -657,7 +663,10 @@ async function startCompassElectron(
let browser: CompassBrowser;

try {
browser = (await remote(options)) as CompassBrowser;
// webdriverio's type is wrong for
// options.capabilities['wdio:chromedriverOptions'] and it doesn't allow
// verbose even though it does work
browser = (await remote(options as any)) as CompassBrowser;
} catch (err) {
debug('Failed to start remote webdriver session', {
error: (err as Error).stack,
Expand Down
45 changes: 45 additions & 0 deletions packages/compass-e2e-tests/installers/helpers.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
import { spawn } from 'child_process';
import type { SpawnOptions } from 'child_process';

export function execute(
command: string,
args: string[],
options?: SpawnOptions
): Promise<void> {
return new Promise((resolve, reject) => {
console.log(command, ...args);
const p = spawn(command, args, {
stdio: 'inherit',
...options,
});
p.on('error', (err: any) => {
reject(err);
});
p.on('close', (code: number | null, signal: NodeJS.Signals | null) => {
if (code !== null) {
if (code === 0) {
resolve();
} else {
reject(
new Error(`${command} ${args.join(' ')} exited with code ${code}`)
);
}
} else {
if (signal !== null) {
reject(
new Error(
`${command} ${args.join(' ')} exited with signal ${signal}`
)
);
} else {
// shouldn't happen
reject(
new Error(
`${command} ${args.join(' ')} exited with no code or signal`
)
);
}
}
});
});
}
53 changes: 53 additions & 0 deletions packages/compass-e2e-tests/installers/mac-dmg.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
import path from 'path';
import { existsSync } from 'fs';
import type { InstalledAppInfo, Package } from './types';
import { execute } from './helpers';

export async function installMacDMG(
appName: string,
{ filepath }: Package
): Promise<InstalledAppInfo> {
const fullDestinationPath = `/Applications/${appName}.app`;

if (existsSync(fullDestinationPath)) {
// Would ideally just throw here, but unfortunately in CI the mac
// environments aren't all clean so somewhere we have to remove it anyway.
console.log(`${fullDestinationPath} already exists. Removing.`);
await execute('rm', ['-rf', fullDestinationPath]);
}

await execute('hdiutil', ['attach', filepath]);
try {
await execute('cp', [
'-Rp',
`/Volumes/${appName}/${appName}.app`,
'/Applications',
]);
} finally {
await execute('hdiutil', ['detach', `/Volumes/${appName}`]);
}

// see if the executable will run without being quarantined or similar
await execute(`/Applications/${appName}.app/Contents/MacOS/${appName}`, [
'--version',
]);

if (process.env.HOME) {
const settingsDir = path.resolve(
process.env.HOME,
'Library',
'Application Support',
appName
);

if (existsSync(settingsDir)) {
console.log(`${settingsDir} already exists. Removing.`);
await execute('rm', ['-rf', settingsDir]);
}
}

return Promise.resolve({
appName,
appPath: `/Applications/${appName}.app`,
});
}
9 changes: 9 additions & 0 deletions packages/compass-e2e-tests/installers/types.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
export type Package = {
filename: string;
filepath: string;
};

export type InstalledAppInfo = {
appName: string;
appPath: string;
};
76 changes: 65 additions & 11 deletions packages/compass-e2e-tests/smoke-test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,9 @@ import { hideBin } from 'yargs/helpers';
import https from 'https';
import { pick } from 'lodash';
import { handler as writeBuildInfo } from 'hadron-build/commands/info';
import type { InstalledAppInfo, Package } from './installers/types';
import { installMacDMG } from './installers/mac-dmg';
import { execute } from './installers/helpers';

const argv = yargs(hideBin(process.argv))
.scriptName('smoke-tests')
Expand Down Expand Up @@ -137,6 +140,10 @@ async function run() {
writeBuildInfo(infoArgs);
const buildInfo = JSON.parse(await fs.readFile(infoArgs.out, 'utf8'));

if (!buildInfoIsCommon(buildInfo)) {
throw new Error('buildInfo is missing');
}

// filter the extensions given the platform (isWindows, isOSX, isUbuntu, isRHEL) and extension
const { isWindows, isOSX, isRHEL, isUbuntu, extension } = context;

Expand All @@ -150,9 +157,9 @@ async function run() {

if (!context.skipDownload) {
await Promise.all(
packages.map(async ({ name, filepath }) => {
packages.map(async ({ filename, filepath }) => {
await fs.mkdir(path.dirname(filepath), { recursive: true });
const url = `https://${context.bucketName}.s3.amazonaws.com/${context.bucketKeyPrefix}/${name}`;
const url = `https://${context.bucketName}.s3.amazonaws.com/${context.bucketKeyPrefix}/${filename}`;
console.log(url);
return downloadFile(url, filepath);
})
Expand All @@ -162,6 +169,24 @@ async function run() {
verifyPackagesExist(packages);

// TODO(COMPASS-8533): extract or install each package and then test the Compass binary
for (const pkg of packages) {
let appInfo: InstalledAppInfo | undefined = undefined;

console.log('installing', pkg.filepath);

if (pkg.filename.endsWith('.dmg')) {
appInfo = await installMacDMG(buildInfo.productName, pkg);
}

// TODO: all the other installers go here

if (appInfo) {
console.log('testing', appInfo.appPath);
await testInstalledApp(appInfo);
} else {
console.log(`no app got installed for ${pkg.filename}`);
}
}
}

function platformFromContext(
Expand Down Expand Up @@ -189,6 +214,18 @@ type PackageFilterConfig = Pick<

// subsets of the hadron-build info result

const commonKeys = ['productName'];
type CommonBuildInfo = Record<typeof commonKeys[number], string>;

function buildInfoIsCommon(buildInfo: any): buildInfo is CommonBuildInfo {
for (const key of commonKeys) {
if (!buildInfo[key]) {
return false;
}
}
return true;
}

const windowsFilenameKeys = [
'windows_setup_filename',
'windows_msi_filename',
Expand Down Expand Up @@ -245,11 +282,6 @@ function buildInfoIsRHEL(buildInfo: any): buildInfo is RHELBuildInfo {
return true;
}

type Package = {
name: string;
filepath: string;
};

function getFilteredPackages(
compassDir: string,
buildInfo: any,
Expand Down Expand Up @@ -282,11 +314,11 @@ function getFilteredPackages(
const extension = config.extension;

return names
.filter((name) => !extension || name.endsWith(extension))
.map((name) => {
.filter((filename) => !extension || filename.endsWith(extension))
.map((filename) => {
return {
name,
filepath: path.join(compassDir, 'dist', name),
filename,
filepath: path.join(compassDir, 'dist', filename),
};
});
}
Expand Down Expand Up @@ -333,6 +365,28 @@ function verifyPackagesExist(packages: Package[]): void {
}
}

function testInstalledApp(appInfo: InstalledAppInfo): Promise<void> {
return execute(
'npm',
[
'run',
'--unsafe-perm',
'test-packaged',
'--workspace',
'compass-e2e-tests',
'--',
'--test-filter=time-to-first-query',
],
{
env: {
...process.env,
COMPASS_APP_NAME: appInfo.appName,
COMPASS_APP_PATH: appInfo.appPath,
},
}
);
}

run()
.then(function () {
console.log('done');
Expand Down

0 comments on commit 6bf3d97

Please sign in to comment.