diff --git a/.evergreen/buildvariants-and-tasks.in.yml b/.evergreen/buildvariants-and-tasks.in.yml index b9370851955..2659cf057fc 100644 --- a/.evergreen/buildvariants-and-tasks.in.yml +++ b/.evergreen/buildvariants-and-tasks.in.yml @@ -80,18 +80,18 @@ const SMOKETEST_BUILD_VARIANTS = [ // run_on: 'rhel80-large', // depends_on: 'package-rhel', // }, -// { -// name: 'smoketest-macos-x64', -// display_name: 'Smoketest MacOS Intel', -// run_on: 'macos-14', -// depends_on: 'package-macos-x64', -// }, -// { -// name: 'smoketest-macos-arm', -// display_name: 'Smoketest MacOS Arm64', -// run_on: 'macos-14-arm64', -// depends_on: 'package-macos-arm', -// } + { + name: 'smoketest-macos-x64', + display_name: 'Smoketest MacOS Intel', + run_on: 'macos-14-gui', + depends_on: 'package-macos-x64', + }, + { + name: 'smoketest-macos-arm', + display_name: 'Smoketest MacOS Arm64', + run_on: 'macos-14-arm64-gui', + depends_on: 'package-macos-arm', + } ]; const TEST_PACKAGED_APP_BUILD_VARIANTS = [ diff --git a/.evergreen/buildvariants-and-tasks.yml b/.evergreen/buildvariants-and-tasks.yml index b5987af6c34..3c79e271bf0 100644 --- a/.evergreen/buildvariants-and-tasks.yml +++ b/.evergreen/buildvariants-and-tasks.yml @@ -84,6 +84,22 @@ buildvariants: variant: package-ubuntu tasks: - name: smoketest-compass + - name: smoketest-macos-x64-compass + display_name: Smoketest MacOS Intel (compass) + run_on: macos-14-gui + depends_on: + - name: package-compass + variant: package-macos-x64 + tasks: + - name: smoketest-compass + - name: smoketest-macos-arm-compass + display_name: Smoketest MacOS Arm64 (compass) + run_on: macos-14-arm64-gui + depends_on: + - name: package-compass + variant: package-macos-arm + tasks: + - name: smoketest-compass - name: test-eol-servers display_name: Test EoL Servers run_on: ubuntu1804-large diff --git a/.evergreen/functions.yml b/.evergreen/functions.yml index 245309cfac5..87690679e90 100644 --- a/.evergreen/functions.yml +++ b/.evergreen/functions.yml @@ -674,7 +674,7 @@ functions: export COMPASS_E2E_DISABLE_CLIPBOARD_USAGE="true" fi - npm run --workspace compass-e2e-tests smoketest + npm run --unsafe-perm --workspace compass-e2e-tests smoketest test-web-sandbox: - command: shell.exec diff --git a/packages/compass-e2e-tests/.prettierignore b/packages/compass-e2e-tests/.prettierignore index 104b57fc109..42c9c0cb109 100644 --- a/packages/compass-e2e-tests/.prettierignore +++ b/packages/compass-e2e-tests/.prettierignore @@ -3,3 +3,4 @@ fixtures .nyc_output coverage +hadron-build-info.json diff --git a/packages/compass-e2e-tests/helpers/chrome-startup-flags.ts b/packages/compass-e2e-tests/helpers/chrome-startup-flags.ts index fb0fc4d3733..fc7ac6dc1b6 100644 --- a/packages/compass-e2e-tests/helpers/chrome-startup-flags.ts +++ b/packages/compass-e2e-tests/helpers/chrome-startup-flags.ts @@ -1,5 +1,6 @@ // Copied from https://github.com/webdriverio/webdriverio/blob/1825c633aead82bc650dff1f403ac30cff7c7cb3/packages/devtools/src/constants.ts // These are the default flags that webdriverio uses to start Chrome driver. +// NOTE: this has since been removed along with the devtools automation protocol https://github.com/webdriverio/webdriverio/commit/28e64e439ffc36a95f24aeda9f1d21111429dfa3#diff-6ea151d6c0687197931735239f397b7f5f0140a588c5b2b82ff584bbe73be069 const DEFAULT_WEBDRIVER_FLAGS = [ // suppresses Save Password prompt window '--enable-automation', diff --git a/packages/compass-e2e-tests/helpers/compass.ts b/packages/compass-e2e-tests/helpers/compass.ts index 317823b09d0..5f3255ef4ec 100644 --- a/packages/compass-e2e-tests/helpers/compass.ts +++ b/packages/compass-e2e-tests/helpers/compass.ts @@ -547,7 +547,7 @@ async function processCommonOpts({ // https://webdriver.io/docs/options/#webdriver-options const webdriverOptions = { - logLevel: 'warn' as const, // info is super verbose right now + logLevel: 'trace' as const, outputDir: webdriverLogPath, }; @@ -601,7 +601,9 @@ async function startCompassElectron( // See https://www.electronjs.org/docs/latest/api/command-line-switches#--enable-loggingfile '--enable-logging=file', // See https://www.electronjs.org/docs/latest/api/command-line-switches#--log-filepath - `--log-file=${electronLogFile}` + `--log-file=${electronLogFile}`, + // See https://chromium.googlesource.com/chromium/src/+/master/docs/chrome_os_logging.md + '--log-level=0' ); if (opts.extraSpawnArgs) { @@ -643,9 +645,13 @@ async function startCompassElectron( }, // from https://github.com/webdriverio-community/wdio-electron-service/blob/32457f60382cb4970c37c7f0a19f2907aaa32443/packages/wdio-electron-service/src/launcher.ts#L102 'wdio:enforceWebDriverClassic': true, - }, - 'wdio:chromedriverOptions': { - // TODO: enable logging so we don't have to debug things blindly + 'wdio:chromedriverOptions': { + // enable logging so we don't have to debug things blindly + // This goes in .log/webdriver/wdio-chromedriver-*.log. It is the + // chromedriver log and since this is verbose it also contains the + // stdout of the electron main process. + verbose: true, + }, }, ...webdriverOptions, ...wdioOptions, @@ -657,7 +663,10 @@ async function startCompassElectron( let browser: CompassBrowser; try { - browser = (await remote(options)) as CompassBrowser; + // webdriverio's type is wrong for + // options.capabilities['wdio:chromedriverOptions'] and it doesn't allow + // verbose even though it does work + browser = (await remote(options as any)) as CompassBrowser; } catch (err) { debug('Failed to start remote webdriver session', { error: (err as Error).stack, diff --git a/packages/compass-e2e-tests/installers/helpers.ts b/packages/compass-e2e-tests/installers/helpers.ts new file mode 100644 index 00000000000..b1fb872f97d --- /dev/null +++ b/packages/compass-e2e-tests/installers/helpers.ts @@ -0,0 +1,45 @@ +import { spawn } from 'child_process'; +import type { SpawnOptions } from 'child_process'; + +export function execute( + command: string, + args: string[], + options?: SpawnOptions +): Promise { + return new Promise((resolve, reject) => { + console.log(command, ...args); + const p = spawn(command, args, { + stdio: 'inherit', + ...options, + }); + p.on('error', (err: any) => { + reject(err); + }); + p.on('close', (code: number | null, signal: NodeJS.Signals | null) => { + if (code !== null) { + if (code === 0) { + resolve(); + } else { + reject( + new Error(`${command} ${args.join(' ')} exited with code ${code}`) + ); + } + } else { + if (signal !== null) { + reject( + new Error( + `${command} ${args.join(' ')} exited with signal ${signal}` + ) + ); + } else { + // shouldn't happen + reject( + new Error( + `${command} ${args.join(' ')} exited with no code or signal` + ) + ); + } + } + }); + }); +} diff --git a/packages/compass-e2e-tests/installers/mac-dmg.ts b/packages/compass-e2e-tests/installers/mac-dmg.ts new file mode 100644 index 00000000000..25883dcc598 --- /dev/null +++ b/packages/compass-e2e-tests/installers/mac-dmg.ts @@ -0,0 +1,53 @@ +import path from 'path'; +import { existsSync } from 'fs'; +import type { InstalledAppInfo, Package } from './types'; +import { execute } from './helpers'; + +export async function installMacDMG( + appName: string, + { filepath }: Package +): Promise { + const fullDestinationPath = `/Applications/${appName}.app`; + + if (existsSync(fullDestinationPath)) { + // Would ideally just throw here, but unfortunately in CI the mac + // environments aren't all clean so somewhere we have to remove it anyway. + console.log(`${fullDestinationPath} already exists. Removing.`); + await execute('rm', ['-rf', fullDestinationPath]); + } + + await execute('hdiutil', ['attach', filepath]); + try { + await execute('cp', [ + '-Rp', + `/Volumes/${appName}/${appName}.app`, + '/Applications', + ]); + } finally { + await execute('hdiutil', ['detach', `/Volumes/${appName}`]); + } + + // see if the executable will run without being quarantined or similar + await execute(`/Applications/${appName}.app/Contents/MacOS/${appName}`, [ + '--version', + ]); + + if (process.env.HOME) { + const settingsDir = path.resolve( + process.env.HOME, + 'Library', + 'Application Support', + appName + ); + + if (existsSync(settingsDir)) { + console.log(`${settingsDir} already exists. Removing.`); + await execute('rm', ['-rf', settingsDir]); + } + } + + return Promise.resolve({ + appName, + appPath: `/Applications/${appName}.app`, + }); +} diff --git a/packages/compass-e2e-tests/installers/types.ts b/packages/compass-e2e-tests/installers/types.ts new file mode 100644 index 00000000000..70855624aa3 --- /dev/null +++ b/packages/compass-e2e-tests/installers/types.ts @@ -0,0 +1,9 @@ +export type Package = { + filename: string; + filepath: string; +}; + +export type InstalledAppInfo = { + appName: string; + appPath: string; +}; diff --git a/packages/compass-e2e-tests/smoke-test.ts b/packages/compass-e2e-tests/smoke-test.ts index 67a9cd2de64..f7f4a5b2258 100755 --- a/packages/compass-e2e-tests/smoke-test.ts +++ b/packages/compass-e2e-tests/smoke-test.ts @@ -7,6 +7,9 @@ import { hideBin } from 'yargs/helpers'; import https from 'https'; import { pick } from 'lodash'; import { handler as writeBuildInfo } from 'hadron-build/commands/info'; +import type { InstalledAppInfo, Package } from './installers/types'; +import { installMacDMG } from './installers/mac-dmg'; +import { execute } from './installers/helpers'; const argv = yargs(hideBin(process.argv)) .scriptName('smoke-tests') @@ -137,6 +140,10 @@ async function run() { writeBuildInfo(infoArgs); const buildInfo = JSON.parse(await fs.readFile(infoArgs.out, 'utf8')); + if (!buildInfoIsCommon(buildInfo)) { + throw new Error('buildInfo is missing'); + } + // filter the extensions given the platform (isWindows, isOSX, isUbuntu, isRHEL) and extension const { isWindows, isOSX, isRHEL, isUbuntu, extension } = context; @@ -150,9 +157,9 @@ async function run() { if (!context.skipDownload) { await Promise.all( - packages.map(async ({ name, filepath }) => { + packages.map(async ({ filename, filepath }) => { await fs.mkdir(path.dirname(filepath), { recursive: true }); - const url = `https://${context.bucketName}.s3.amazonaws.com/${context.bucketKeyPrefix}/${name}`; + const url = `https://${context.bucketName}.s3.amazonaws.com/${context.bucketKeyPrefix}/${filename}`; console.log(url); return downloadFile(url, filepath); }) @@ -162,6 +169,24 @@ async function run() { verifyPackagesExist(packages); // TODO(COMPASS-8533): extract or install each package and then test the Compass binary + for (const pkg of packages) { + let appInfo: InstalledAppInfo | undefined = undefined; + + console.log('installing', pkg.filepath); + + if (pkg.filename.endsWith('.dmg')) { + appInfo = await installMacDMG(buildInfo.productName, pkg); + } + + // TODO: all the other installers go here + + if (appInfo) { + console.log('testing', appInfo.appPath); + await testInstalledApp(appInfo); + } else { + console.log(`no app got installed for ${pkg.filename}`); + } + } } function platformFromContext( @@ -189,6 +214,18 @@ type PackageFilterConfig = Pick< // subsets of the hadron-build info result +const commonKeys = ['productName']; +type CommonBuildInfo = Record; + +function buildInfoIsCommon(buildInfo: any): buildInfo is CommonBuildInfo { + for (const key of commonKeys) { + if (!buildInfo[key]) { + return false; + } + } + return true; +} + const windowsFilenameKeys = [ 'windows_setup_filename', 'windows_msi_filename', @@ -245,11 +282,6 @@ function buildInfoIsRHEL(buildInfo: any): buildInfo is RHELBuildInfo { return true; } -type Package = { - name: string; - filepath: string; -}; - function getFilteredPackages( compassDir: string, buildInfo: any, @@ -282,11 +314,11 @@ function getFilteredPackages( const extension = config.extension; return names - .filter((name) => !extension || name.endsWith(extension)) - .map((name) => { + .filter((filename) => !extension || filename.endsWith(extension)) + .map((filename) => { return { - name, - filepath: path.join(compassDir, 'dist', name), + filename, + filepath: path.join(compassDir, 'dist', filename), }; }); } @@ -333,6 +365,28 @@ function verifyPackagesExist(packages: Package[]): void { } } +function testInstalledApp(appInfo: InstalledAppInfo): Promise { + return execute( + 'npm', + [ + 'run', + '--unsafe-perm', + 'test-packaged', + '--workspace', + 'compass-e2e-tests', + '--', + '--test-filter=time-to-first-query', + ], + { + env: { + ...process.env, + COMPASS_APP_NAME: appInfo.appName, + COMPASS_APP_PATH: appInfo.appPath, + }, + } + ); +} + run() .then(function () { console.log('done');