feat(e2e): smoketest macOS .dmg files COMPASS-8709

.evergreen/buildvariants-and-tasks.in.yml

@@ -80,18 +80,18 @@ const SMOKETEST_BUILD_VARIANTS = [
 //    run_on: 'rhel80-large',
 //    depends_on: 'package-rhel',
 //  },
-//  {
-//    name: 'smoketest-macos-x64',
-//    display_name: 'Smoketest MacOS Intel',
-//    run_on: 'macos-14',
-//    depends_on: 'package-macos-x64',
-//  },
-//  {
-//    name: 'smoketest-macos-arm',
-//    display_name: 'Smoketest MacOS Arm64',
-//    run_on: 'macos-14-arm64',
-//    depends_on: 'package-macos-arm',
-//  }
+  {
+    name: 'smoketest-macos-x64',
+    display_name: 'Smoketest MacOS Intel',
+    run_on: 'macos-14-gui',
+    depends_on: 'package-macos-x64',
+  },
+  {
+    name: 'smoketest-macos-arm',
+    display_name: 'Smoketest MacOS Arm64',
+    run_on: 'macos-14-arm64-gui',
+    depends_on: 'package-macos-arm',
+  }
 ];
 
 const TEST_PACKAGED_APP_BUILD_VARIANTS = [

.evergreen/buildvariants-and-tasks.yml

@@ -84,6 +84,22 @@ buildvariants:
         variant: package-ubuntu
     tasks:
       - name: smoketest-compass
+  - name: smoketest-macos-x64-compass
+    display_name: Smoketest MacOS Intel (compass)
+    run_on: macos-14-gui
+    depends_on:
+      - name: package-compass
+        variant: package-macos-x64
+    tasks:
+      - name: smoketest-compass
+  - name: smoketest-macos-arm-compass
+    display_name: Smoketest MacOS Arm64 (compass)
+    run_on: macos-14-arm64-gui
+    depends_on:
+      - name: package-compass
+        variant: package-macos-arm
+    tasks:
+      - name: smoketest-compass
   - name: test-eol-servers
     display_name: Test EoL Servers
     run_on: ubuntu1804-large

.evergreen/functions.yml

@@ -674,7 +674,7 @@ functions:
             export COMPASS_E2E_DISABLE_CLIPBOARD_USAGE="true"
           fi
 
-          npm run --workspace compass-e2e-tests smoketest
+          npm run --unsafe-perm --workspace compass-e2e-tests smoketest
 
   test-web-sandbox:
     - command: shell.exec

packages/compass-e2e-tests/.prettierignore

@@ -3,3 +3,4 @@
 fixtures
 .nyc_output
 coverage
+hadron-build-info.json

packages/compass-e2e-tests/helpers/chrome-startup-flags.ts

@@ -1,5 +1,6 @@
 // Copied from https://github.com/webdriverio/webdriverio/blob/1825c633aead82bc650dff1f403ac30cff7c7cb3/packages/devtools/src/constants.ts
 // These are the default flags that webdriverio uses to start Chrome driver.
+// NOTE: this has since been removed along with the devtools automation protocol https://github.com/webdriverio/webdriverio/commit/28e64e439ffc36a95f24aeda9f1d21111429dfa3#diff-6ea151d6c0687197931735239f397b7f5f0140a588c5b2b82ff584bbe73be069
 const DEFAULT_WEBDRIVER_FLAGS = [
   // suppresses Save Password prompt window
   '--enable-automation',

packages/compass-e2e-tests/helpers/compass.ts

@@ -547,7 +547,7 @@ async function processCommonOpts({
 
   // https://webdriver.io/docs/options/#webdriver-options
   const webdriverOptions = {
-    logLevel: 'warn' as const, // info is super verbose right now
+    logLevel: 'trace' as const,
     outputDir: webdriverLogPath,
   };
 
@@ -601,7 +601,9 @@ async function startCompassElectron(
     // See https://www.electronjs.org/docs/latest/api/command-line-switches#--enable-loggingfile
     '--enable-logging=file',
     // See https://www.electronjs.org/docs/latest/api/command-line-switches#--log-filepath
-    `--log-file=${electronLogFile}`
+    `--log-file=${electronLogFile}`,
+    // See https://chromium.googlesource.com/chromium/src/+/master/docs/chrome_os_logging.md
+    '--log-level=0'
   );
 
   if (opts.extraSpawnArgs) {
@@ -643,9 +645,13 @@ async function startCompassElectron(
       },
       // from https://github.com/webdriverio-community/wdio-electron-service/blob/32457f60382cb4970c37c7f0a19f2907aaa32443/packages/wdio-electron-service/src/launcher.ts#L102
       'wdio:enforceWebDriverClassic': true,
-    },
-    'wdio:chromedriverOptions': {
-      // TODO: enable logging so we don't have to debug things blindly
+      'wdio:chromedriverOptions': {
+        // enable logging so we don't have to debug things blindly
+        // This goes in .log/webdriver/wdio-chromedriver-*.log. It is the
+        // chromedriver log and since this is verbose it also contains the
+        // stdout of the electron main process.
+        verbose: true,
+      },
     },
     ...webdriverOptions,
     ...wdioOptions,
@@ -657,7 +663,10 @@ async function startCompassElectron(
   let browser: CompassBrowser;
 
   try {
-    browser = (await remote(options)) as CompassBrowser;
+    // webdriverio's type is wrong for
+    // options.capabilities['wdio:chromedriverOptions'] and it doesn't allow
+    // verbose even though it does work
+    browser = (await remote(options as any)) as CompassBrowser;
   } catch (err) {
     debug('Failed to start remote webdriver session', {
       error: (err as Error).stack,

packages/compass-e2e-tests/installers/helpers.ts

@@ -0,0 +1,45 @@
+import { spawn } from 'child_process';
+import type { SpawnOptions } from 'child_process';
+
+export function execute(
+  command: string,
+  args: string[],
+  options?: SpawnOptions
+): Promise<void> {
+  return new Promise((resolve, reject) => {
+    console.log(command, ...args);
+    const p = spawn(command, args, {
+      stdio: 'inherit',
+      ...options,
+    });
+    p.on('error', (err: any) => {
+      reject(err);
+    });
+    p.on('close', (code: number | null, signal: NodeJS.Signals | null) => {
+      if (code !== null) {
+        if (code === 0) {
+          resolve();
+        } else {
+          reject(
+            new Error(`${command} ${args.join(' ')} exited with code ${code}`)
+          );
+        }
+      } else {
+        if (signal !== null) {
+          reject(
+            new Error(
+              `${command} ${args.join(' ')} exited with signal ${signal}`
+            )
+          );
+        } else {
+          // shouldn't happen
+          reject(
+            new Error(
+              `${command} ${args.join(' ')} exited with no code or signal`
+            )
+          );
+        }
+      }
+    });
+  });
+}

packages/compass-e2e-tests/installers/mac-dmg.ts

@@ -0,0 +1,53 @@
+import path from 'path';
+import { existsSync } from 'fs';
+import type { InstalledAppInfo, Package } from './types';
+import { execute } from './helpers';
+
+export async function installMacDMG(
+  appName: string,
+  { filepath }: Package
+): Promise<InstalledAppInfo> {
+  const fullDestinationPath = `/Applications/${appName}.app`;
+
+  if (existsSync(fullDestinationPath)) {
+    // Would ideally just throw here, but unfortunately in CI the mac
+    // environments aren't all clean so somewhere we have to remove it anyway.
+    console.log(`${fullDestinationPath} already exists. Removing.`);
+    await execute('rm', ['-rf', fullDestinationPath]);
+  }
+
+  await execute('hdiutil', ['attach', filepath]);
+  try {
+    await execute('cp', [
+      '-Rp',
+      `/Volumes/${appName}/${appName}.app`,
+      '/Applications',
+    ]);
+  } finally {
+    await execute('hdiutil', ['detach', `/Volumes/${appName}`]);
+  }
+
+  // see if the executable will run without being quarantined or similar
+  await execute(`/Applications/${appName}.app/Contents/MacOS/${appName}`, [
+    '--version',
+  ]);
+
+  if (process.env.HOME) {
+    const settingsDir = path.resolve(
+      process.env.HOME,
+      'Library',
+      'Application Support',
+      appName
+    );
+
+    if (existsSync(settingsDir)) {
+      console.log(`${settingsDir} already exists. Removing.`);
+      await execute('rm', ['-rf', settingsDir]);
+    }
+  }
+
+  return Promise.resolve({
+    appName,
+    appPath: `/Applications/${appName}.app`,
+  });
+}

packages/compass-e2e-tests/installers/types.ts

@@ -0,0 +1,9 @@
+export type Package = {
+  filename: string;
+  filepath: string;
+};
+
+export type InstalledAppInfo = {
+  appName: string;
+  appPath: string;
+};

packages/compass-e2e-tests/smoke-test.ts

@@ -7,6 +7,9 @@ import { hideBin } from 'yargs/helpers';
 import https from 'https';
 import { pick } from 'lodash';
 import { handler as writeBuildInfo } from 'hadron-build/commands/info';
+import type { InstalledAppInfo, Package } from './installers/types';
+import { installMacDMG } from './installers/mac-dmg';
+import { execute } from './installers/helpers';
 
 const argv = yargs(hideBin(process.argv))
   .scriptName('smoke-tests')
@@ -137,6 +140,10 @@ async function run() {
   writeBuildInfo(infoArgs);
   const buildInfo = JSON.parse(await fs.readFile(infoArgs.out, 'utf8'));
 
+  if (!buildInfoIsCommon(buildInfo)) {
+    throw new Error('buildInfo is missing');
+  }
+
   // filter the extensions given the platform (isWindows, isOSX, isUbuntu, isRHEL) and extension
   const { isWindows, isOSX, isRHEL, isUbuntu, extension } = context;
 
@@ -150,9 +157,9 @@ async function run() {
 
   if (!context.skipDownload) {
     await Promise.all(
-      packages.map(async ({ name, filepath }) => {
+      packages.map(async ({ filename, filepath }) => {
         await fs.mkdir(path.dirname(filepath), { recursive: true });
-        const url = `https://${context.bucketName}.s3.amazonaws.com/${context.bucketKeyPrefix}/${name}`;
+        const url = `https://${context.bucketName}.s3.amazonaws.com/${context.bucketKeyPrefix}/${filename}`;
         console.log(url);
         return downloadFile(url, filepath);
       })
@@ -162,6 +169,24 @@ async function run() {
   verifyPackagesExist(packages);
 
   // TODO(COMPASS-8533): extract or install each package and then test the Compass binary
+  for (const pkg of packages) {
+    let appInfo: InstalledAppInfo | undefined = undefined;
+
+    console.log('installing', pkg.filepath);
+
+    if (pkg.filename.endsWith('.dmg')) {
+      appInfo = await installMacDMG(buildInfo.productName, pkg);
+    }
+
+    // TODO: all the other installers go here
+
+    if (appInfo) {
+      console.log('testing', appInfo.appPath);
+      await testInstalledApp(appInfo);
+    } else {
+      console.log(`no app got installed for ${pkg.filename}`);
+    }
+  }
 }
 
 function platformFromContext(
@@ -189,6 +214,18 @@ type PackageFilterConfig = Pick<
 
 // subsets of the hadron-build info result
 
+const commonKeys = ['productName'];
+type CommonBuildInfo = Record<typeof commonKeys[number], string>;
+
+function buildInfoIsCommon(buildInfo: any): buildInfo is CommonBuildInfo {
+  for (const key of commonKeys) {
+    if (!buildInfo[key]) {
+      return false;
+    }
+  }
+  return true;
+}
+
 const windowsFilenameKeys = [
   'windows_setup_filename',
   'windows_msi_filename',
@@ -245,11 +282,6 @@ function buildInfoIsRHEL(buildInfo: any): buildInfo is RHELBuildInfo {
   return true;
 }
 
-type Package = {
-  name: string;
-  filepath: string;
-};
-
 function getFilteredPackages(
   compassDir: string,
   buildInfo: any,
@@ -282,11 +314,11 @@ function getFilteredPackages(
   const extension = config.extension;
 
   return names
-    .filter((name) => !extension || name.endsWith(extension))
-    .map((name) => {
+    .filter((filename) => !extension || filename.endsWith(extension))
+    .map((filename) => {
       return {
-        name,
-        filepath: path.join(compassDir, 'dist', name),
+        filename,
+        filepath: path.join(compassDir, 'dist', filename),
       };
     });
 }
@@ -333,6 +365,28 @@ function verifyPackagesExist(packages: Package[]): void {
   }
 }
 
+function testInstalledApp(appInfo: InstalledAppInfo): Promise<void> {
+  return execute(
+    'npm',
+    [
+      'run',
+      '--unsafe-perm',
+      'test-packaged',
+      '--workspace',
+      'compass-e2e-tests',
+      '--',
+      '--test-filter=time-to-first-query',
+    ],
+    {
+      env: {
+        ...process.env,
+        COMPASS_APP_NAME: appInfo.appName,
+        COMPASS_APP_PATH: appInfo.appPath,
+      },
+    }
+  );
+}
+
 run()
   .then(function () {
     console.log('done');