[7.x] [Reporting] code cleanup for reporting browser build/install/se…

…tup utilities (#98799) (#98998)

* [Reporting] code cleanup for reporting browser build/install/setup utilities (#98799)

* [Reporting] code cleanup for reporting browser setup utilities

* fix target_cpu

* Update README.md

* Update README.md

* add note about target_cpu

* Update paths.ts

* more cleanup

* Update src/dev/chromium_version.ts

Co-authored-by: Michael Dokolin <[email protected]>

* remove bug

Co-authored-by: Michael Dokolin <[email protected]>
Co-authored-by: Kibana Machine <[email protected]>
# Conflicts:
#	src/dev/code_coverage/ingest_coverage/__tests__/enumerate_patterns.test.js
#	src/dev/code_coverage/ingest_coverage/__tests__/mocks/team_assign_mock.txt
#	src/dev/code_coverage/ingest_coverage/__tests__/transforms.test.js

* fix bad merge

Co-authored-by: Kibana Machine <[email protected]>

src/dev/chromium_version.ts

@@ -35,8 +35,10 @@ async function getPuppeteerRelease(log: ToolingLog): Promise<PuppeteerRelease> {
       'Could not get the Puppeteer version! Check node_modules/puppteer/package.json'
     );
   }
-  log.info(`Kibana is using Puppeteer ${version} (${forkCompatibilityMap[version]})`);
-  return forkCompatibilityMap[version];
+  const puppeteerRelease = forkCompatibilityMap[version] ?? version;
+
+  log.info(`Kibana is using Puppeteer ${version} (${puppeteerRelease})`);
+  return puppeteerRelease;
 }
 
 async function getChromiumRevision(
@@ -129,8 +131,8 @@ run(
     description: chalk`
       Display the Chromium git commit that correlates to a given Puppeteer release.
 
-      -  node x-pack/dev-tools/chromium_version 5.5.0  {dim # gets the Chromium commit for Puppeteer v5.5.0}
-      -  node x-pack/dev-tools/chromium_version       {dim  # gets the Chromium commit for the Kibana dependency version of Puppeteer}
+      -  node scripts/chromium_version 5.5.0  {dim # gets the Chromium commit for Puppeteer v5.5.0}
+      -  node scripts/chromium_version       {dim  # gets the Chromium commit for the Kibana dependency version of Puppeteer}
 
       You can use https://omahaproxy.appspot.com/ to look up the Chromium release that first shipped with that commit.
     `,

x-pack/.gitignore

@@ -3,7 +3,7 @@
 /target
 /test/functional/failure_debug
 /test/functional/screenshots
-/test/functional/apps/reporting/reports/session
+/test/functional/apps/**/reports/session
 /test/reporting/configs/failure_debug/
 /plugins/reporting/.chromium/
 /plugins/reporting/chromium/

x-pack/build_chromium/README.md

@@ -12,8 +12,8 @@ which is where we have two machines provisioned for the Linux and Windows
 builds. Mac builds can be achieved locally, and are a great place to start to
 gain familiarity.
 
-**NOTE:** Linux builds should be done in Ubuntu on x86 architecture. ARM builds
-are created in x86. CentOS is not supported for building Chromium.
+**NOTE:** Linux builds should be done in Ubuntu on x64 architecture. ARM builds
+are created in x64 using cross-compiling. CentOS is not supported for building Chromium.
 
 1. Login to our GCP instance [here using your okta credentials](https://console.cloud.google.com/).
 2. Click the "Compute Engine" tab.
@@ -27,25 +27,32 @@ are created in x86. CentOS is not supported for building Chromium.
     - python2 (`python` must link to `python2`)
     - lsb_release
     - tmux is recommended in case your ssh session is interrupted
-6. Copy the entire `build_chromium` directory into a GCP storage bucket, so you can copy the scripts into the instance and run them.
+    - "Cloud API access scopes": must have **read / write** scope for the Storage API
+6. Copy the entire `build_chromium` directory from the `headless_shell_staging` bucket. To do this, use `gsutil rsync`:
+   ```sh
+   # This shows a preview of what would change by synchronizing the source scripts with the destination GCS bucket.
+   # Remove the `-n` flag to enact the changes
+   gsutil -m rsync -n -r x-pack/build_chromium gs://headless_shell_staging/build_chromium
+   ```
 
 ## Build Script Usage
 
-```
+These commands show how to set up an environment to build:
+```sh
 # Allow our scripts to use depot_tools commands
 export PATH=$HOME/chromium/depot_tools:$PATH
 
 # Create a dedicated working directory for this directory of Python scripts.
 mkdir ~/chromium && cd ~/chromium
 
-# Copy the scripts from the Kibana repo to use them conveniently in the working directory
-gsutil cp -r gs://my-bucket/build_chromium .
+# Copy the scripts from the Kibana team's GCS bucket
+gsutil cp -r gs://headless_shell_staging/build_chromium .
 
 # Install the OS packages, configure the environment, download the chromium source (25GB)
-python ./build_chromium/init.sh [arch_name]
+python ./build_chromium/init.py [arch_name]
 
 # Run the build script with the path to the chromium src directory, the git commit hash
-python ./build_chromium/build.py <commit_id> x86
+python ./build_chromium/build.py <commit_id> x64
 
 # OR You can build for ARM
 python ./build_chromium/build.py <commit_id> arm64
@@ -107,7 +114,7 @@ use the Kibana `build.py` script (in this directory).
 
 It's recommended that you create a working directory for the chromium source
 code and all the build tools, and run the commands from there:
-```
+```sh
 mkdir ~/chromium && cd ~/chromium
 cp -r ~/path/to/kibana/x-pack/build_chromium .
 python ./build_chromium/init.sh [arch_name]
@@ -216,6 +223,7 @@ In the case of Windows, you can use IE to open `http://localhost:9221` and see i
 
 The following links provide helpful context about how the Chromium build works, and its prerequisites:
 
+- Tools for Chromium version information: https://omahaproxy.appspot.com/
 - https://www.chromium.org/developers/how-tos/get-the-code/working-with-release-branches
 - https://chromium.googlesource.com/chromium/src/+/HEAD/docs/windows_build_instructions.md
 - https://chromium.googlesource.com/chromium/src/+/HEAD/docs/mac_build_instructions.md

x-pack/build_chromium/build.py

@@ -3,21 +3,18 @@
 from build_util import (
   runcmd,
   runcmdsilent,
-  mkdir,
   md5_file,
-  configure_environment,
 )
 
 # This file builds Chromium headless on Windows, Mac, and Linux.
 
 # Verify that we have an argument, and if not print instructions
 if (len(sys.argv) < 2):
   print('Usage:')
-  print('python build.py {chromium_version} [arch_name]')
+  print('python build.py {chromium_version} {arch_name}')
   print('Example:')
-  print('python build.py 68.0.3440.106')
-  print('python build.py 4747cc23ae334a57a35ed3c8e6adcdbc8a50d479')
-  print('python build.py 4747cc23ae334a57a35ed3c8e6adcdbc8a50d479 arm64 # build for ARM architecture')
+  print('python build.py 4747cc23ae334a57a35ed3c8e6adcdbc8a50d479 x64')
+  print('python build.py 4747cc23ae334a57a35ed3c8e6adcdbc8a50d479 arm64 # cross-compile for ARM architecture')
   print
   sys.exit(1)
 
@@ -57,25 +54,34 @@
   print('Creating a new branch for tracking the source version')
   runcmd('git checkout -b build-' + base_version + ' ' + source_version)
 
+# configure environment: environment path
 depot_tools_path = os.path.join(build_path, 'depot_tools')
-path_value = depot_tools_path + os.pathsep + os.environ['PATH']
-print('Updating PATH for depot_tools: ' + path_value)
-os.environ['PATH'] = path_value
+full_path = depot_tools_path + os.pathsep + os.environ['PATH']
+print('Updating PATH for depot_tools: ' + full_path)
+os.environ['PATH'] = full_path
+
+# configure environment: build dependencies
+if platform.system() == 'Linux':
+  if arch_name:
+    print('Running sysroot install script...')
+    runcmd(src_path + '/build/linux/sysroot_scripts/install-sysroot.py --arch=' + arch_name)
+  print('Running install-build-deps...')
+  runcmd(src_path + '/build/install-build-deps.sh')
+
+
 print('Updating all modules')
-runcmd('gclient sync')
+runcmd('gclient sync -D')
 
-# Copy build args/{Linux | Darwin | Windows}.gn from the root of our directory to out/headless/args.gn,
-argsgn_destination = path.abspath('out/headless/args.gn')
-print('Generating platform-specific args')
-mkdir('out/headless')
-print(' > cp ' + argsgn_file + ' ' + argsgn_destination)
-shutil.copyfile(argsgn_file, argsgn_destination)
+print('Setting up build directory')
+runcmd('rm -rf out/headless')
+runcmd('mkdir out/headless')
 
+# Copy build args/{Linux | Darwin | Windows}.gn from the root of our directory to out/headless/args.gn,
+# add the target_cpu for cross-compilation
 print('Adding target_cpu to args')
-
-f = open('out/headless/args.gn', 'a')
-f.write('\rtarget_cpu = "' + arch_name + '"\r')
-f.close()
+argsgn_file_out = path.abspath('out/headless/args.gn')
+runcmd('cp ' + argsgn_file + ' ' + argsgn_file_out)
+runcmd('echo \'target_cpu="' + arch_name + '"\' >> ' + argsgn_file_out)
 
 runcmd('gn gen out/headless')
 
@@ -136,3 +142,6 @@ def archive_file(name):
 print('Creating ' + path.join(src_path, md5_filename))
 with open (md5_filename, 'w') as f:
   f.write(md5_file(zip_filename))
+
+runcmd('gsutil cp ' + path.join(src_path, zip_filename) + ' gs://headless_shell_staging')
+runcmd('gsutil cp ' + path.join(src_path, md5_filename) + ' gs://headless_shell_staging')

x-pack/build_chromium/build_util.py

@@ -27,19 +27,3 @@ def md5_file(filename):
     for chunk in iter(lambda: f.read(128 * md5.block_size), b''):
       md5.update(chunk)
   return md5.hexdigest()
-
-def configure_environment(arch_name, build_path, src_path):
-  """Runs install scripts for deps, and configures temporary environment variables required by Chromium's build"""
-
-  if platform.system() == 'Linux':
-    if arch_name:
-      print('Running sysroot install script...')
-      sysroot_cmd = src_path + '/build/linux/sysroot_scripts/install-sysroot.py --arch=' + arch_name
-      runcmd(sysroot_cmd)
-    print('Running install-build-deps...')
-    runcmd(src_path + '/build/install-build-deps.sh')
-
-  depot_tools_path = os.path.join(build_path, 'depot_tools')
-  full_path = depot_tools_path + os.pathsep + os.environ['PATH']
-  print('Updating PATH for depot_tools: ' + full_path)
-  os.environ['PATH'] = full_path

x-pack/build_chromium/darwin/args.gn

@@ -19,13 +19,15 @@ use_alsa = false
 use_cups = false
 use_dbus = false
 use_gio = false
-# Please, consult @elastic/kibana-security before changing/removing this option.
-use_kerberos = false
 use_libpci = false
 use_pulseaudio = false
 use_udev = false
 
 is_debug = false
 symbol_level = 0
 is_component_build = false
-remove_webcore_debug_symbols = true
+
+# Please, consult @elastic/kibana-security before changing/removing this option.
+use_kerberos = false
+
+# target_cpu is appended before build: "x64" or "arm64"

x-pack/build_chromium/init.py

@@ -1,6 +1,6 @@
 import os, platform, sys
 from os import path
-from build_util import runcmd, mkdir, md5_file, configure_environment
+from build_util import runcmd, mkdir
 
 # This is a cross-platform initialization script which should only be run
 # once per environment, and isn't intended to be run directly. You should
@@ -44,6 +44,3 @@
   runcmd('fetch chromium --nohooks=1 --no-history=1')
 else:
   print('Directory exists: ' + chromium_dir + '. Skipping chromium fetch.')
-
-# This depends on having the chromium/src directory with the complete checkout
-configure_environment(arch_name, build_path, src_path)

x-pack/build_chromium/linux/args.gn

@@ -2,7 +2,8 @@ import("//build/args/headless.gn")
 is_debug = false
 symbol_level = 0
 is_component_build = false
-remove_webcore_debug_symbols = true
 enable_nacl = false
 # Please, consult @elastic/kibana-security before changing/removing this option.
 use_kerberos = false
+
+# target_cpu is appended before build: "x64" or "arm64"

x-pack/build_chromium/windows/args.gn

@@ -18,10 +18,12 @@ use_gio = false
 use_libpci = false
 use_pulseaudio = false
 use_udev = false
-# Please, consult @elastic/kibana-security before changing/removing this option.
-use_kerberos = false
 
 is_debug = false
 symbol_level = 0
 is_component_build = false
-remove_webcore_debug_symbols = true
+
+# Please, consult @elastic/kibana-security before changing/removing this option.
+use_kerberos = false
+
+# target_cpu is appended before build: "x64" or "arm64"

x-pack/plugins/reporting/server/browsers/chromium/driver/chromium_driver.ts

@@ -8,7 +8,7 @@
 import { i18n } from '@kbn/i18n';
 import { map, truncate } from 'lodash';
 import open from 'opn';
-import { ElementHandle, EvaluateFn, Page, Response, SerializableOrJSHandle } from 'puppeteer';
+import puppeteer, { ElementHandle, EvaluateFn, SerializableOrJSHandle } from 'puppeteer';
 import { parse as parseUrl } from 'url';
 import { getDisallowedOutgoingUrlError } from '../';
 import { ConditionalHeaders, ConditionalHeadersConditions } from '../../../export_types/common';
@@ -53,14 +53,14 @@ interface InterceptedRequest {
 const WAIT_FOR_DELAY_MS: number = 100;
 
 export class HeadlessChromiumDriver {
-  private readonly page: Page;
+  private readonly page: puppeteer.Page;
   private readonly inspect: boolean;
   private readonly networkPolicy: NetworkPolicy;
 
   private listenersAttached = false;
   private interceptedCount = 0;
 
-  constructor(page: Page, { inspect, networkPolicy }: ChromiumDriverOptions) {
+  constructor(page: puppeteer.Page, { inspect, networkPolicy }: ChromiumDriverOptions) {
     this.page = page;
     this.inspect = inspect;
     this.networkPolicy = networkPolicy;
@@ -127,7 +127,7 @@ export class HeadlessChromiumDriver {
   /*
    * Call Page.screenshot and return a base64-encoded string of the image
    */
-  public async screenshot(elementPosition: ElementPosition): Promise<string> {
+  public async screenshot(elementPosition: ElementPosition): Promise<string | void> {
     const { boundingClientRect, scroll } = elementPosition;
     const screenshot = await this.page.screenshot({
       clip: {
@@ -138,7 +138,10 @@ export class HeadlessChromiumDriver {
       },
     });
 
-    return screenshot.toString('base64');
+    if (screenshot) {
+      return screenshot.toString('base64');
+    }
+    return screenshot;
   }
 
   public async evaluate(
@@ -160,6 +163,11 @@ export class HeadlessChromiumDriver {
     const { timeout } = opts;
     logger.debug(`waitForSelector ${selector}`);
     const resp = await this.page.waitForSelector(selector, { timeout }); // override default 30000ms
+
+    if (!resp) {
+      throw new Error(`Failure in waitForSelector: void response! Context: ${context.context}`);
+    }
+
     logger.debug(`waitForSelector ${selector} resolved`);
     return resp;
   }
@@ -219,6 +227,7 @@ export class HeadlessChromiumDriver {
     }
 
     // @ts-ignore
+    // FIXME: use `await page.target().createCDPSession();`
     const client = this.page._client;
 
     // We have to reach into the Chrome Devtools Protocol to apply headers as using
@@ -293,7 +302,7 @@ export class HeadlessChromiumDriver {
     // Even though 3xx redirects go through our request
     // handler, we should probably inspect responses just to
     // avoid being bamboozled by some malicious request
-    this.page.on('response', (interceptedResponse: Response) => {
+    this.page.on('response', (interceptedResponse: puppeteer.Response) => {
       const interceptedUrl = interceptedResponse.url();
       const allowed = !interceptedUrl.startsWith('file://');
 
@@ -315,17 +324,17 @@ export class HeadlessChromiumDriver {
 
   private async launchDebugger() {
     // In order to pause on execution we have to reach more deeply into Chromiums Devtools Protocol,
-    // and more specifically, for the page being used. _client is per-page, and puppeteer doesn't expose
-    // a page's client in their api, so we have to reach into internals to get this behavior.
-    // Finally, in order to get the inspector running, we have to know the page's internal ID (again, private)
+    // and more specifically, for the page being used. _client is per-page.
+    // In order to get the inspector running, we have to know the page's internal ID (again, private)
     // in order to construct the final debugging URL.
 
+    const target = this.page.target();
+    const client = await target.createCDPSession();
+
+    await client.send('Debugger.enable');
+    await client.send('Debugger.pause');
     // @ts-ignore
-    await this.page._client.send('Debugger.enable');
-    // @ts-ignore
-    await this.page._client.send('Debugger.pause');
-    // @ts-ignore
-    const targetId = this.page._target._targetId;
+    const targetId = target._targetId;
     const wsEndpoint = this.page.browser().wsEndpoint();
     const { port } = parseUrl(wsEndpoint);
 

x-pack/plugins/reporting/server/browsers/chromium/driver_factory/index.ts

@@ -194,6 +194,10 @@ export class HeadlessChromiumDriverFactory {
     // Puppeteer doesn't give a handle to the original ChildProcess object
     // See https://github.com/GoogleChrome/puppeteer/issues/1292#issuecomment-521470627
 
+    if (childProcess == null) {
+      throw new TypeError('childProcess is null or undefined!');
+    }
+
     // just log closing of the process
     const processClose$ = Rx.fromEvent<void>(childProcess, 'close').pipe(
       tap(() => {