[Reporting] code cleanup for reporting browser build/install/setup ut…

…ilities (#98799)

* [Reporting] code cleanup for reporting browser setup utilities

* fix target_cpu

* Update README.md

* Update README.md

* add note about target_cpu

* Update paths.ts

* more cleanup

* Update src/dev/chromium_version.ts

Co-authored-by: Michael Dokolin <[email protected]>

* remove bug

Co-authored-by: Michael Dokolin <[email protected]>
Co-authored-by: Kibana Machine <[email protected]>

src/dev/chromium_version.ts

@@ -35,8 +35,10 @@ async function getPuppeteerRelease(log: ToolingLog): Promise<PuppeteerRelease> {
       'Could not get the Puppeteer version! Check node_modules/puppteer/package.json'
     );
   }
-  log.info(`Kibana is using Puppeteer ${version} (${forkCompatibilityMap[version]})`);
-  return forkCompatibilityMap[version];
+  const puppeteerRelease = forkCompatibilityMap[version] ?? version;
+
+  log.info(`Kibana is using Puppeteer ${version} (${puppeteerRelease})`);
+  return puppeteerRelease;
 }
 
 async function getChromiumRevision(
@@ -129,8 +131,8 @@ run(
     description: chalk`
       Display the Chromium git commit that correlates to a given Puppeteer release.
 
-      -  node x-pack/dev-tools/chromium_version 5.5.0  {dim # gets the Chromium commit for Puppeteer v5.5.0}
-      -  node x-pack/dev-tools/chromium_version       {dim  # gets the Chromium commit for the Kibana dependency version of Puppeteer}
+      -  node scripts/chromium_version 5.5.0  {dim # gets the Chromium commit for Puppeteer v5.5.0}
+      -  node scripts/chromium_version       {dim  # gets the Chromium commit for the Kibana dependency version of Puppeteer}
 
       You can use https://omahaproxy.appspot.com/ to look up the Chromium release that first shipped with that commit.
     `,

src/dev/code_coverage/ingest_coverage/__tests__/enumerate_patterns.test.js

@@ -15,14 +15,14 @@ const log = new ToolingLog({
 });
 
 describe(`enumeratePatterns`, () => {
-  it(`should resolve x-pack/plugins/reporting/server/browsers/extract/unzip.js to kibana-reporting`, () => {
+  it(`should resolve x-pack/plugins/reporting/server/browsers/extract/unzip.ts to kibana-reporting`, () => {
     const actual = enumeratePatterns(REPO_ROOT)(log)(
       new Map([['x-pack/plugins/reporting', ['kibana-reporting']]])
     );
 
     expect(
       actual[0].includes(
-        'x-pack/plugins/reporting/server/browsers/extract/unzip.js kibana-reporting'
+        'x-pack/plugins/reporting/server/browsers/extract/unzip.ts kibana-reporting'
       )
     ).toBe(true);
   });

src/dev/code_coverage/ingest_coverage/__tests__/mocks/team_assign_mock.txt

@@ -148,10 +148,10 @@ x-pack/plugins/reporting/server/browsers/download/download.ts kibana-reporting
 x-pack/plugins/reporting/server/browsers/download/ensure_downloaded.ts kibana-reporting
 x-pack/plugins/reporting/server/browsers/download/index.ts kibana-reporting
 x-pack/plugins/reporting/server/browsers/download/util.ts kibana-reporting
-x-pack/plugins/reporting/server/browsers/extract/extract.js kibana-reporting
-x-pack/plugins/reporting/server/browsers/extract/extract_error.js kibana-reporting
-x-pack/plugins/reporting/server/browsers/extract/index.js kibana-reporting
-x-pack/plugins/reporting/server/browsers/extract/unzip.js kibana-reporting
+x-pack/plugins/reporting/server/browsers/extract/extract.ts kibana-reporting
+x-pack/plugins/reporting/server/browsers/extract/extract_error.ts kibana-reporting
+x-pack/plugins/reporting/server/browsers/extract/index.ts kibana-reporting
+x-pack/plugins/reporting/server/browsers/extract/unzip.ts kibana-reporting
 x-pack/plugins/reporting/server/browsers/index.ts kibana-reporting
 x-pack/plugins/reporting/server/browsers/install.ts kibana-reporting
 x-pack/plugins/reporting/server/browsers/network_policy.test.ts kibana-reporting

src/dev/code_coverage/ingest_coverage/__tests__/transforms.test.js

@@ -32,27 +32,27 @@ describe(`Transform fns`, () => {
       it(`should remove the jenkins workspace path`, () => {
         const obj = {
           staticSiteUrl:
-            '/var/lib/jenkins/workspace/elastic+kibana+code-coverage/kibana/x-pack/plugins/reporting/server/browsers/extract/unzip.js',
+            '/var/lib/jenkins/workspace/elastic+kibana+code-coverage/kibana/x-pack/plugins/reporting/server/browsers/extract/unzip.ts',
           COVERAGE_INGESTION_KIBANA_ROOT:
             '/var/lib/jenkins/workspace/elastic+kibana+code-coverage/kibana',
         };
         expect(coveredFilePath(obj)).toHaveProperty(
           'coveredFilePath',
-          'x-pack/plugins/reporting/server/browsers/extract/unzip.js'
+          'x-pack/plugins/reporting/server/browsers/extract/unzip.ts'
         );
       });
     });
     describe(`in the qa research job`, () => {
       it(`should remove the jenkins workspace path`, () => {
         const obj = {
           staticSiteUrl:
-            '/var/lib/jenkins/workspace/elastic+kibana+qa-research/kibana/x-pack/plugins/reporting/server/browsers/extract/unzip.js',
+            '/var/lib/jenkins/workspace/elastic+kibana+qa-research/kibana/x-pack/plugins/reporting/server/browsers/extract/unzip.ts',
           COVERAGE_INGESTION_KIBANA_ROOT:
             '/var/lib/jenkins/workspace/elastic+kibana+qa-research/kibana',
         };
         expect(coveredFilePath(obj)).toHaveProperty(
           'coveredFilePath',
-          'x-pack/plugins/reporting/server/browsers/extract/unzip.js'
+          'x-pack/plugins/reporting/server/browsers/extract/unzip.ts'
         );
       });
     });
@@ -82,7 +82,7 @@ describe(`Transform fns`, () => {
   describe(`teamAssignment`, () => {
     const teamAssignmentsPathMOCK =
       'src/dev/code_coverage/ingest_coverage/__tests__/mocks/team_assign_mock.txt';
-    const coveredFilePath = 'x-pack/plugins/reporting/server/browsers/extract/unzip.js';
+    const coveredFilePath = 'x-pack/plugins/reporting/server/browsers/extract/unzip.ts';
     const obj = { coveredFilePath };
     const log = new ToolingLog({
       level: 'info',

x-pack/.gitignore

@@ -3,7 +3,7 @@
 /target
 /test/functional/failure_debug
 /test/functional/screenshots
-/test/functional/apps/reporting/reports/session
+/test/functional/apps/**/reports/session
 /test/reporting/configs/failure_debug/
 /plugins/reporting/.chromium/
 /plugins/reporting/chromium/

x-pack/build_chromium/README.md

@@ -12,8 +12,8 @@ which is where we have two machines provisioned for the Linux and Windows
 builds. Mac builds can be achieved locally, and are a great place to start to
 gain familiarity.
 
-**NOTE:** Linux builds should be done in Ubuntu on x86 architecture. ARM builds
-are created in x86. CentOS is not supported for building Chromium.
+**NOTE:** Linux builds should be done in Ubuntu on x64 architecture. ARM builds
+are created in x64 using cross-compiling. CentOS is not supported for building Chromium.
 
 1. Login to our GCP instance [here using your okta credentials](https://console.cloud.google.com/).
 2. Click the "Compute Engine" tab.
@@ -27,25 +27,32 @@ are created in x86. CentOS is not supported for building Chromium.
     - python2 (`python` must link to `python2`)
     - lsb_release
     - tmux is recommended in case your ssh session is interrupted
-6. Copy the entire `build_chromium` directory into a GCP storage bucket, so you can copy the scripts into the instance and run them.
+    - "Cloud API access scopes": must have **read / write** scope for the Storage API
+6. Copy the entire `build_chromium` directory from the `headless_shell_staging` bucket. To do this, use `gsutil rsync`:
+   ```sh
+   # This shows a preview of what would change by synchronizing the source scripts with the destination GCS bucket.
+   # Remove the `-n` flag to enact the changes
+   gsutil -m rsync -n -r x-pack/build_chromium gs://headless_shell_staging/build_chromium
+   ```
 
 ## Build Script Usage
 
-```
+These commands show how to set up an environment to build:
+```sh
 # Allow our scripts to use depot_tools commands
 export PATH=$HOME/chromium/depot_tools:$PATH
 
 # Create a dedicated working directory for this directory of Python scripts.
 mkdir ~/chromium && cd ~/chromium
 
-# Copy the scripts from the Kibana repo to use them conveniently in the working directory
-gsutil cp -r gs://my-bucket/build_chromium .
+# Copy the scripts from the Kibana team's GCS bucket
+gsutil cp -r gs://headless_shell_staging/build_chromium .
 
 # Install the OS packages, configure the environment, download the chromium source (25GB)
-python ./build_chromium/init.sh [arch_name]
+python ./build_chromium/init.py [arch_name]
 
 # Run the build script with the path to the chromium src directory, the git commit hash
-python ./build_chromium/build.py <commit_id> x86
+python ./build_chromium/build.py <commit_id> x64
 
 # OR You can build for ARM
 python ./build_chromium/build.py <commit_id> arm64
@@ -107,7 +114,7 @@ use the Kibana `build.py` script (in this directory).
 
 It's recommended that you create a working directory for the chromium source
 code and all the build tools, and run the commands from there:
-```
+```sh
 mkdir ~/chromium && cd ~/chromium
 cp -r ~/path/to/kibana/x-pack/build_chromium .
 python ./build_chromium/init.sh [arch_name]
@@ -216,6 +223,7 @@ In the case of Windows, you can use IE to open `http://localhost:9221` and see i
 
 The following links provide helpful context about how the Chromium build works, and its prerequisites:
 
+- Tools for Chromium version information: https://omahaproxy.appspot.com/
 - https://www.chromium.org/developers/how-tos/get-the-code/working-with-release-branches
 - https://chromium.googlesource.com/chromium/src/+/HEAD/docs/windows_build_instructions.md
 - https://chromium.googlesource.com/chromium/src/+/HEAD/docs/mac_build_instructions.md

x-pack/build_chromium/build.py

@@ -3,21 +3,18 @@
 from build_util import (
   runcmd,
   runcmdsilent,
-  mkdir,
   md5_file,
-  configure_environment,
 )
 
 # This file builds Chromium headless on Windows, Mac, and Linux.
 
 # Verify that we have an argument, and if not print instructions
 if (len(sys.argv) < 2):
   print('Usage:')
-  print('python build.py {chromium_version} [arch_name]')
+  print('python build.py {chromium_version} {arch_name}')
   print('Example:')
-  print('python build.py 68.0.3440.106')
-  print('python build.py 4747cc23ae334a57a35ed3c8e6adcdbc8a50d479')
-  print('python build.py 4747cc23ae334a57a35ed3c8e6adcdbc8a50d479 arm64 # build for ARM architecture')
+  print('python build.py 4747cc23ae334a57a35ed3c8e6adcdbc8a50d479 x64')
+  print('python build.py 4747cc23ae334a57a35ed3c8e6adcdbc8a50d479 arm64 # cross-compile for ARM architecture')
   print
   sys.exit(1)
 
@@ -57,25 +54,34 @@
   print('Creating a new branch for tracking the source version')
   runcmd('git checkout -b build-' + base_version + ' ' + source_version)
 
+# configure environment: environment path
 depot_tools_path = os.path.join(build_path, 'depot_tools')
-path_value = depot_tools_path + os.pathsep + os.environ['PATH']
-print('Updating PATH for depot_tools: ' + path_value)
-os.environ['PATH'] = path_value
+full_path = depot_tools_path + os.pathsep + os.environ['PATH']
+print('Updating PATH for depot_tools: ' + full_path)
+os.environ['PATH'] = full_path
+
+# configure environment: build dependencies
+if platform.system() == 'Linux':
+  if arch_name:
+    print('Running sysroot install script...')
+    runcmd(src_path + '/build/linux/sysroot_scripts/install-sysroot.py --arch=' + arch_name)
+  print('Running install-build-deps...')
+  runcmd(src_path + '/build/install-build-deps.sh')
+
+
 print('Updating all modules')
-runcmd('gclient sync')
+runcmd('gclient sync -D')
 
-# Copy build args/{Linux | Darwin | Windows}.gn from the root of our directory to out/headless/args.gn,
-argsgn_destination = path.abspath('out/headless/args.gn')
-print('Generating platform-specific args')
-mkdir('out/headless')
-print(' > cp ' + argsgn_file + ' ' + argsgn_destination)
-shutil.copyfile(argsgn_file, argsgn_destination)
+print('Setting up build directory')
+runcmd('rm -rf out/headless')
+runcmd('mkdir out/headless')
 
+# Copy build args/{Linux | Darwin | Windows}.gn from the root of our directory to out/headless/args.gn,
+# add the target_cpu for cross-compilation
 print('Adding target_cpu to args')
-
-f = open('out/headless/args.gn', 'a')
-f.write('\rtarget_cpu = "' + arch_name + '"\r')
-f.close()
+argsgn_file_out = path.abspath('out/headless/args.gn')
+runcmd('cp ' + argsgn_file + ' ' + argsgn_file_out)
+runcmd('echo \'target_cpu="' + arch_name + '"\' >> ' + argsgn_file_out)
 
 runcmd('gn gen out/headless')
 
@@ -136,3 +142,6 @@ def archive_file(name):
 print('Creating ' + path.join(src_path, md5_filename))
 with open (md5_filename, 'w') as f:
   f.write(md5_file(zip_filename))
+
+runcmd('gsutil cp ' + path.join(src_path, zip_filename) + ' gs://headless_shell_staging')
+runcmd('gsutil cp ' + path.join(src_path, md5_filename) + ' gs://headless_shell_staging')

x-pack/build_chromium/build_util.py

@@ -27,19 +27,3 @@ def md5_file(filename):
     for chunk in iter(lambda: f.read(128 * md5.block_size), b''):
       md5.update(chunk)
   return md5.hexdigest()
-
-def configure_environment(arch_name, build_path, src_path):
-  """Runs install scripts for deps, and configures temporary environment variables required by Chromium's build"""
-
-  if platform.system() == 'Linux':
-    if arch_name:
-      print('Running sysroot install script...')
-      sysroot_cmd = src_path + '/build/linux/sysroot_scripts/install-sysroot.py --arch=' + arch_name
-      runcmd(sysroot_cmd)
-    print('Running install-build-deps...')
-    runcmd(src_path + '/build/install-build-deps.sh')
-
-  depot_tools_path = os.path.join(build_path, 'depot_tools')
-  full_path = depot_tools_path + os.pathsep + os.environ['PATH']
-  print('Updating PATH for depot_tools: ' + full_path)
-  os.environ['PATH'] = full_path

x-pack/build_chromium/darwin/args.gn

@@ -19,13 +19,15 @@ use_alsa = false
 use_cups = false
 use_dbus = false
 use_gio = false
-# Please, consult @elastic/kibana-security before changing/removing this option.
-use_kerberos = false
 use_libpci = false
 use_pulseaudio = false
 use_udev = false
 
 is_debug = false
 symbol_level = 0
 is_component_build = false
-remove_webcore_debug_symbols = true
+
+# Please, consult @elastic/kibana-security before changing/removing this option.
+use_kerberos = false
+
+# target_cpu is appended before build: "x64" or "arm64"

x-pack/build_chromium/init.py

@@ -1,6 +1,6 @@
 import os, platform, sys
 from os import path
-from build_util import runcmd, mkdir, md5_file, configure_environment
+from build_util import runcmd, mkdir
 
 # This is a cross-platform initialization script which should only be run
 # once per environment, and isn't intended to be run directly. You should
@@ -44,6 +44,3 @@
   runcmd('fetch chromium --nohooks=1 --no-history=1')
 else:
   print('Directory exists: ' + chromium_dir + '. Skipping chromium fetch.')
-
-# This depends on having the chromium/src directory with the complete checkout
-configure_environment(arch_name, build_path, src_path)

x-pack/build_chromium/linux/args.gn

@@ -2,7 +2,8 @@ import("//build/args/headless.gn")
 is_debug = false
 symbol_level = 0
 is_component_build = false
-remove_webcore_debug_symbols = true
 enable_nacl = false
 # Please, consult @elastic/kibana-security before changing/removing this option.
 use_kerberos = false
+
+# target_cpu is appended before build: "x64" or "arm64"

x-pack/build_chromium/windows/args.gn

@@ -18,10 +18,12 @@ use_gio = false
 use_libpci = false
 use_pulseaudio = false
 use_udev = false
-# Please, consult @elastic/kibana-security before changing/removing this option.
-use_kerberos = false
 
 is_debug = false
 symbol_level = 0
 is_component_build = false
-remove_webcore_debug_symbols = true
+
+# Please, consult @elastic/kibana-security before changing/removing this option.
+use_kerberos = false
+
+# target_cpu is appended before build: "x64" or "arm64"