[build] Creates Linux aarch64 archive (#69165)

- Updates Linux Chromium builds to accept architecture argument (defaults to x64) for arm64 support. - Example: `python ~/chromium/build_chromium/build.py 312d84c8ce62810976feda0d3457108a6dfff9e6 arm64` - Updates all Chromium builds to include architecture in filename. - `chromium-312d84c-linux_arm64.zip` _(new)_ - `chromium-312d84c-linux.zip` > `chromium-312d84c-linux_x64.zip` - Moves Chromium install from data directory to `x-pack/plugins/reporting/chromium` - Moves Chromium download cache from `x-pack/plugins/reporting/.chromium` to `.chromium` - Installs Chromium during build (closes #53664) - Updates build to be architecture aware (x64 and aarch64) - Removed Chromium debug logs, they were not helpful and can not be written inside the Kibana root. If we were to keep them, we would need to write to `logging.dist`. Signed-off-by: Tyler Smalley <[email protected]>
elastic · Jul 10, 2020 · afe785b · afe785b
1 parent d8e9327
commit afe785b
Show file tree

Hide file tree

Showing 35 changed files with 219 additions and 255 deletions.
diff --git a/.ci/packer_cache_for_branch.sh b/.ci/packer_cache_for_branch.sh
@@ -46,7 +46,7 @@ echo "Creating bootstrap_cache archive"
 # archive cacheable directories
 mkdir -p "$HOME/.kibana/bootstrap_cache"
 tar -cf "$HOME/.kibana/bootstrap_cache/$branch.tar" \
-  x-pack/plugins/reporting/.chromium \
+  .chromium \
   .es \
   .chromedriver \
   .geckodriver;

diff --git a/.eslintignore b/.eslintignore
@@ -1,6 +1,7 @@
 **/*.js.snap
 **/graphql/types.ts
 /.es
+/.chromium
 /build
 /built_assets
 /config/apm.dev.js

diff --git a/.gitignore b/.gitignore
@@ -2,6 +2,7 @@
 .signing-config.json
 .ackrc
 /.es
+/.chromium
 .DS_Store
 .node_binaries
 .native_modules

diff --git a/docs/user/reporting/index.asciidoc b/docs/user/reporting/index.asciidoc
@@ -19,7 +19,7 @@ image::user/reporting/images/share-button.png["Share"]
 [float]
 == Setup
 
-{reporting} is automatically enabled in {kib}. The first time {kib} runs, it extracts a custom build for the Chromium web browser, which
+{reporting} is automatically enabled in {kib}. It runs a custom build of the Chromium web browser, which
 runs on the server in headless mode to load {kib} and capture the rendered {kib} charts as images.
 
 Chromium is an open-source project not related to Elastic, but the Chromium binary for {kib} has been custom-built by Elastic to ensure it

diff --git a/src/cli/cluster/cluster_manager.ts b/src/cli/cluster/cluster_manager.ts
@@ -261,7 +261,7 @@ export class ClusterManager {
       /debug\.log$/,
       ...pluginInternalDirsIgnore,
       fromRoot('src/legacy/server/sass/__tmp__'),
-      fromRoot('x-pack/plugins/reporting/.chromium'),
+      fromRoot('x-pack/plugins/reporting/chromium'),
       fromRoot('x-pack/plugins/security_solution/cypress'),
       fromRoot('x-pack/plugins/apm/e2e'),
       fromRoot('x-pack/plugins/apm/scripts'),

diff --git a/src/dev/build/README.md b/src/dev/build/README.md
@@ -24,7 +24,7 @@ The majority of this logic is extracted from the grunt build that has existed fo
 
 **Config**: [lib/config.js] defines the config used to execute tasks. It is mostly used to determine absolute paths to specific locations, and to get access to the Platforms.
 
-**Platform**: [lib/platform.js] defines the Platform objects, which define the different platforms we build for. Use `config.getTargetPlatforms()` to get the list of platforms we are targeting in this build, `config.getNodePlatforms()` to get the list of platform we will download node for, or `config.getLinux/Windows/MacPlatform()` to get a specific platform.
+**Platform**: [lib/platform.js] defines the Platform objects, which define the different platforms we build for. Use `config.getTargetPlatforms()` to get the list of platforms we are targeting in this build, `config.getNodePlatforms()` to get the list of platform we will download node for, or `config.getPlatform` to get a specific platform and architecture.
 
 **Log**: We uses the `ToolingLog` defined in [../tooling_log/tooling_log.js]
 

diff --git a/src/dev/build/build_distributables.js b/src/dev/build/build_distributables.js
@@ -20,16 +20,16 @@
 import { getConfig, createRunner } from './lib';
 
 import {
+  BuildKibanaPlatformPluginsTask,
   BuildPackagesTask,
   CleanClientModulesOnDLLTask,
   CleanEmptyFoldersTask,
   CleanExtraBinScriptsTask,
-  CleanExtraBrowsersTask,
   CleanExtraFilesFromModulesTask,
-  CleanPackagesTask,
-  CleanTypescriptTask,
   CleanNodeBuildsTask,
+  CleanPackagesTask,
   CleanTask,
+  CleanTypescriptTask,
   CopyBinScriptsTask,
   CopySourceTask,
   CreateArchivesSourcesTask,
@@ -44,20 +44,20 @@ import {
   CreateRpmPackageTask,
   DownloadNodeBuildsTask,
   ExtractNodeBuildsTask,
+  InstallChromiumTask,
   InstallDependenciesTask,
-  BuildKibanaPlatformPluginsTask,
   OptimizeBuildTask,
   PatchNativeModulesTask,
+  PathLengthTask,
   RemovePackageJsonDepsTask,
   RemoveWorkspacesTask,
   TranspileBabelTask,
   TranspileScssTask,
   UpdateLicenseFileTask,
+  UuidVerificationTask,
   VerifyEnvTask,
   VerifyExistingNodeBuildsTask,
-  PathLengthTask,
   WriteShaSumsTask,
-  UuidVerificationTask,
 } from './tasks';
 
 export async function buildDistributables(options) {
@@ -134,12 +134,12 @@ export async function buildDistributables(options) {
 
   /**
    * copy generic build outputs into platform-specific build
-   * directories and perform platform-specific steps
+   * directories and perform platform/architecture-specific steps
    */
   await run(CreateArchivesSourcesTask);
   await run(PatchNativeModulesTask);
+  await run(InstallChromiumTask);
   await run(CleanExtraBinScriptsTask);
-  await run(CleanExtraBrowsersTask);
   await run(CleanNodeBuildsTask);
 
   await run(PathLengthTask);

diff --git a/src/dev/build/lib/__tests__/config.js b/src/dev/build/lib/__tests__/config.js
@@ -72,15 +72,31 @@ describe('dev/build/lib/config', () => {
     });
   });
 
+  describe('#getPlatform()', () => {
+    it('throws error when platform does not exist', async () => {
+      const { config } = await setup();
+      const fn = () => config.getPlatform('foo', 'x64');
+
+      expect(fn).to.throwException(/Unable to find platform/);
+    });
+
+    it('throws error when architecture does not exist', async () => {
+      const { config } = await setup();
+      const fn = () => config.getPlatform('linux', 'foo');
+
+      expect(fn).to.throwException(/Unable to find platform/);
+    });
+  });
+
   describe('#getTargetPlatforms()', () => {
     it('returns an array of all platform objects', async () => {
       const { config } = await setup();
       expect(
         config
           .getTargetPlatforms()
-          .map((p) => p.getName())
+          .map((p) => p.getNodeArch())
           .sort()
-      ).to.eql(['darwin', 'linux', 'windows']);
+      ).to.eql(['darwin-x64', 'linux-arm64', 'linux-x64', 'win32-x64']);
     });
 
     it('returns just this platform when targetAllPlatforms = false', async () => {
@@ -99,9 +115,9 @@ describe('dev/build/lib/config', () => {
       expect(
         config
           .getTargetPlatforms()
-          .map((p) => p.getName())
+          .map((p) => p.getNodeArch())
           .sort()
-      ).to.eql(['darwin', 'linux', 'windows']);
+      ).to.eql(['darwin-x64', 'linux-arm64', 'linux-x64', 'win32-x64']);
     });
 
     it('returns this platform and linux, when targetAllPlatforms = false', async () => {
@@ -111,39 +127,20 @@ describe('dev/build/lib/config', () => {
       if (process.platform !== 'linux') {
         expect(platforms).to.have.length(2);
         expect(platforms[0]).to.be(config.getPlatformForThisOs());
-        expect(platforms[1]).to.be(config.getLinuxPlatform());
+        expect(platforms[1]).to.be(config.getPlatform('linux', 'x64'));
       } else {
         expect(platforms).to.have.length(1);
-        expect(platforms[0]).to.be(config.getLinuxPlatform());
+        expect(platforms[0]).to.be(config.getPlatform('linux', 'x64'));
       }
     });
   });
 
-  describe('#getLinuxPlatform()', () => {
-    it('returns the linux platform', async () => {
-      const { config } = await setup();
-      expect(config.getLinuxPlatform().getName()).to.be('linux');
-    });
-  });
-
-  describe('#getWindowsPlatform()', () => {
-    it('returns the windows platform', async () => {
-      const { config } = await setup();
-      expect(config.getWindowsPlatform().getName()).to.be('windows');
-    });
-  });
-
-  describe('#getMacPlatform()', () => {
-    it('returns the mac platform', async () => {
-      const { config } = await setup();
-      expect(config.getMacPlatform().getName()).to.be('darwin');
-    });
-  });
-
   describe('#getPlatformForThisOs()', () => {
     it('returns the platform that matches the arch of this machine', async () => {
       const { config } = await setup();
-      expect(config.getPlatformForThisOs().getName()).to.be(process.platform);
+      const currentPlatform = config.getPlatformForThisOs();
+      expect(currentPlatform.getName()).to.be(process.platform);
+      expect(currentPlatform.getArchitecture()).to.be(process.arch);
     });
   });
 

diff --git a/src/dev/build/lib/__tests__/platform.js b/src/dev/build/lib/__tests__/platform.js
@@ -30,37 +30,39 @@ describe('src/dev/build/lib/platform', () => {
 
   describe('getNodeArch()', () => {
     it('returns the node arch for the passed name', () => {
-      expect(createPlatform('windows').getNodeArch()).to.be('windows-x64');
+      expect(createPlatform('win32', 'x64').getNodeArch()).to.be('win32-x64');
     });
   });
 
   describe('getBuildName()', () => {
     it('returns the build name for the passed name', () => {
-      expect(createPlatform('windows').getBuildName()).to.be('windows-x86_64');
+      expect(createPlatform('linux', 'arm64', 'linux-aarch64').getBuildName()).to.be(
+        'linux-aarch64'
+      );
     });
   });
 
   describe('isWindows()', () => {
-    it('returns true if name is windows', () => {
-      expect(createPlatform('windows').isWindows()).to.be(true);
-      expect(createPlatform('linux').isWindows()).to.be(false);
-      expect(createPlatform('darwin').isWindows()).to.be(false);
+    it('returns true if name is win32', () => {
+      expect(createPlatform('win32', 'x64').isWindows()).to.be(true);
+      expect(createPlatform('linux', 'x64').isWindows()).to.be(false);
+      expect(createPlatform('darwin', 'x64').isWindows()).to.be(false);
     });
   });
 
   describe('isLinux()', () => {
     it('returns true if name is linux', () => {
-      expect(createPlatform('windows').isLinux()).to.be(false);
-      expect(createPlatform('linux').isLinux()).to.be(true);
-      expect(createPlatform('darwin').isLinux()).to.be(false);
+      expect(createPlatform('win32', 'x64').isLinux()).to.be(false);
+      expect(createPlatform('linux', 'x64').isLinux()).to.be(true);
+      expect(createPlatform('darwin', 'x64').isLinux()).to.be(false);
     });
   });
 
   describe('isMac()', () => {
     it('returns true if name is darwin', () => {
-      expect(createPlatform('windows').isMac()).to.be(false);
-      expect(createPlatform('linux').isMac()).to.be(false);
-      expect(createPlatform('darwin').isMac()).to.be(true);
+      expect(createPlatform('win32', 'x64').isMac()).to.be(false);
+      expect(createPlatform('linux', 'x64').isMac()).to.be(false);
+      expect(createPlatform('darwin', 'x64').isMac()).to.be(true);
     });
   });
 });
diff --git a/src/dev/build/lib/config.js b/src/dev/build/lib/config.js
@@ -18,7 +18,7 @@
  */
 
 import { dirname, resolve, relative } from 'path';
-import { platform as getOsPlatform } from 'os';
+import os from 'os';
 
 import { getVersionInfo } from './version_info';
 import { createPlatform } from './platform';
@@ -29,7 +29,12 @@ export async function getConfig({ isRelease, targetAllPlatforms, versionQualifie
   const repoRoot = dirname(pkgPath);
   const nodeVersion = pkg.engines.node;
 
-  const platforms = ['darwin', 'linux', 'windows'].map(createPlatform);
+  const platforms = [
+    createPlatform('linux', 'x64', 'linux-x86_64'),
+    createPlatform('linux', 'arm64', 'linux-aarch64'),
+    createPlatform('darwin', 'x64', 'darwin-x86_64'),
+    createPlatform('win32', 'x64', 'windows-x86_64'),
+  ];
 
   const versionInfo = await getVersionInfo({
     isRelease,
@@ -101,51 +106,30 @@ export async function getConfig({ isRelease, targetAllPlatforms, versionQualifie
       }
 
       if (process.platform === 'linux') {
-        return [this.getLinuxPlatform()];
+        return [this.getPlatform('linux', 'x64')];
       }
 
-      return [this.getPlatformForThisOs(), this.getLinuxPlatform()];
+      return [this.getPlatformForThisOs(), this.getPlatform('linux', 'x64')];
     }
 
-    /**
-     * Get the linux platform object
-     * @return {Platform}
-     */
-    getLinuxPlatform() {
-      return platforms.find((p) => p.isLinux());
-    }
+    getPlatform(name, arch) {
+      const selected = platforms.find((p) => {
+        return name === p.getName() && arch === p.getArchitecture();
+      });
 
-    /**
-     * Get the windows platform object
-     * @return {Platform}
-     */
-    getWindowsPlatform() {
-      return platforms.find((p) => p.isWindows());
-    }
+      if (!selected) {
+        throw new Error(`Unable to find platform (${name}) with architecture (${arch})`);
+      }
 
-    /**
-     * Get the mac platform object
-     * @return {Platform}
-     */
-    getMacPlatform() {
-      return platforms.find((p) => p.isMac());
+      return selected;
     }
 
     /**
      * Get the platform object representing the OS on this machine
      * @return {Platform}
      */
     getPlatformForThisOs() {
-      switch (getOsPlatform()) {
-        case 'darwin':
-          return this.getMacPlatform();
-        case 'win32':
-          return this.getWindowsPlatform();
-        case 'linux':
-          return this.getLinuxPlatform();
-        default:
-          throw new Error(`Unable to find platform for this os`);
-      }
+      return this.getPlatform(os.platform(), os.arch());
     }
 
     /**

diff --git a/src/dev/build/lib/platform.js b/src/dev/build/lib/platform.js
@@ -17,22 +17,26 @@
  * under the License.
  */
 
-export function createPlatform(name) {
+export function createPlatform(name, architecture, buildName) {
   return new (class Platform {
     getName() {
       return name;
     }
 
-    getNodeArch() {
-      return `${name}-x64`;
+    getArchitecture() {
+      return architecture;
     }
 
     getBuildName() {
-      return `${name}-x86_64`;
+      return buildName;
+    }
+
+    getNodeArch() {
+      return `${name}-${architecture}`;
     }
 
     isWindows() {
-      return name === 'windows';
+      return name === 'win32';
     }
 
     isMac() {