[build] Add cloud docker images

This adds a new build task to create cloud docker images. These images include metricbeat and filebeat. Archives are for both are expected to be in the target folder prior to starting the build. The cloud image can be generated with `node scripts/build --all-platforms --docker-cloud` Closes elastic#99916
jbudz · Aug 9, 2021 · ace7220 · ace7220
1 parent 8d90df3
commit ace7220
Show file tree

Hide file tree

Showing 7 changed files with 65 additions and 0 deletions.
diff --git a/src/dev/build/args.test.ts b/src/dev/build/args.test.ts
@@ -29,6 +29,7 @@ it('build default and oss dist for current platform, without packages, by defaul
         "createArchives": true,
         "createDebPackage": false,
         "createDockerCentOS": false,
+        "createDockerCloud": false,
         "createDockerContexts": true,
         "createDockerUBI": false,
         "createExamplePlugins": false,
@@ -55,6 +56,7 @@ it('builds packages if --all-platforms is passed', () => {
         "createArchives": true,
         "createDebPackage": true,
         "createDockerCentOS": true,
+        "createDockerCloud": false,
         "createDockerContexts": true,
         "createDockerUBI": true,
         "createExamplePlugins": false,
@@ -81,6 +83,7 @@ it('limits packages if --rpm passed with --all-platforms', () => {
         "createArchives": true,
         "createDebPackage": false,
         "createDockerCentOS": false,
+        "createDockerCloud": false,
         "createDockerContexts": true,
         "createDockerUBI": false,
         "createExamplePlugins": false,
@@ -107,6 +110,7 @@ it('limits packages if --deb passed with --all-platforms', () => {
         "createArchives": true,
         "createDebPackage": true,
         "createDockerCentOS": false,
+        "createDockerCloud": false,
         "createDockerContexts": true,
         "createDockerUBI": false,
         "createExamplePlugins": false,
@@ -134,6 +138,7 @@ it('limits packages if --docker passed with --all-platforms', () => {
         "createArchives": true,
         "createDebPackage": false,
         "createDockerCentOS": true,
+        "createDockerCloud": false,
         "createDockerContexts": true,
         "createDockerUBI": true,
         "createExamplePlugins": false,
@@ -168,6 +173,7 @@ it('limits packages if --docker passed with --skip-docker-ubi and --all-platform
         "createArchives": true,
         "createDebPackage": false,
         "createDockerCentOS": true,
+        "createDockerCloud": false,
         "createDockerContexts": true,
         "createDockerUBI": false,
         "createExamplePlugins": false,
@@ -195,6 +201,7 @@ it('limits packages if --all-platforms passed with --skip-docker-centos', () =>
         "createArchives": true,
         "createDebPackage": true,
         "createDockerCentOS": false,
+        "createDockerCloud": false,
         "createDockerContexts": true,
         "createDockerUBI": true,
         "createExamplePlugins": false,

diff --git a/src/dev/build/args.ts b/src/dev/build/args.ts
@@ -26,6 +26,7 @@ export function readCliArgs(argv: string[]) {
       'skip-docker-contexts',
       'skip-docker-ubi',
       'skip-docker-centos',
+      'docker-cloud',
       'release',
       'skip-node-download',
       'verbose',
@@ -103,6 +104,7 @@ export function readCliArgs(argv: string[]) {
     createDebPackage: isOsPackageDesired('deb'),
     createDockerCentOS:
       isOsPackageDesired('docker-images') && !Boolean(flags['skip-docker-centos']),
+    createDockerCloud: isOsPackageDesired('docker-images') && Boolean(flags['docker-cloud']),
     createDockerUBI: isOsPackageDesired('docker-images') && !Boolean(flags['skip-docker-ubi']),
     createDockerContexts: !Boolean(flags['skip-docker-contexts']),
     targetAllPlatforms: Boolean(flags['all-platforms']),

diff --git a/src/dev/build/build_distributables.ts b/src/dev/build/build_distributables.ts
@@ -22,6 +22,7 @@ export interface BuildOptions {
   createDebPackage: boolean;
   createDockerUBI: boolean;
   createDockerCentOS: boolean;
+  createDockerCloud: boolean;
   createDockerContexts: boolean;
   versionQualifier: string | undefined;
   targetAllPlatforms: boolean;
@@ -123,6 +124,11 @@ export async function buildDistributables(log: ToolingLog, options: BuildOptions
     await run(Tasks.CreateDockerCentOS);
   }
 
+  if (options.createDockerCloud) {
+    // control w/ --docker-images and --docker-cloud
+    await run(Tasks.CreateDockerCloud);
+  }
+
   if (options.createDockerContexts) {
     // control w/ --skip-docker-contexts
     await run(Tasks.CreateDockerContexts);

diff --git a/src/dev/build/tasks/os_packages/create_os_package_tasks.ts b/src/dev/build/tasks/os_packages/create_os_package_tasks.ts
@@ -82,6 +82,25 @@ export const CreateDockerUBI: Task = {
   },
 };
 
+export const CreateDockerCloud: Task = {
+  description: 'Creating Docker Cloud image',
+
+  async run(config, log, build) {
+    await runDockerGenerator(config, log, build, {
+      architecture: 'x64',
+      context: false,
+      cloud: true,
+      image: true,
+    });
+    await runDockerGenerator(config, log, build, {
+      architecture: 'aarch64',
+      context: false,
+      cloud: true,
+      image: true,
+    });
+  },
+};
+
 export const CreateDockerContexts: Task = {
   description: 'Creating Docker build contexts',
 

diff --git a/src/dev/build/tasks/os_packages/docker_generator/run.ts b/src/dev/build/tasks/os_packages/docker_generator/run.ts
@@ -33,6 +33,7 @@ export async function runDockerGenerator(
     image: boolean;
     ubi?: boolean;
     ironbank?: boolean;
+    cloud?: boolean;
     dockerBuildDate?: string;
   }
 ) {
@@ -43,6 +44,7 @@ export async function runDockerGenerator(
   let imageFlavor = '';
   if (flags.ubi) imageFlavor += `-${ubiVersionTag}`;
   if (flags.ironbank) imageFlavor += '-ironbank';
+  if (flags.cloud) imageFlavor += '-cloud';
 
   // General docker var config
   const license = 'Elastic License';
@@ -51,6 +53,8 @@ export async function runDockerGenerator(
   const artifactArchitecture = flags.architecture === 'aarch64' ? 'aarch64' : 'x86_64';
   const artifactPrefix = `kibana-${version}-linux`;
   const artifactTarball = `${artifactPrefix}-${artifactArchitecture}.tar.gz`;
+  const metricbeatTarball = `metricbeat-${version}-linux-${artifactArchitecture}.tar.gz`;
+  const filebeatTarball = `filebeat-${version}-linux-${artifactArchitecture}.tar.gz`;
   const artifactsDir = config.resolveFromTarget('.');
   const dockerBuildDate = flags.dockerBuildDate || new Date().toISOString();
   // That would produce oss, default and default-ubi7
@@ -73,6 +77,9 @@ export async function runDockerGenerator(
     baseOSImage,
     dockerBuildDate,
     ubi: flags.ubi,
+    cloud: flags.cloud,
+    metricbeatTarball,
+    filebeatTarball,
     ironbank: flags.ironbank,
     architecture: flags.architecture,
     revision: config.getBuildSha(),
@@ -97,6 +104,8 @@ export async function runDockerGenerator(
     await accessAsync(resolve(artifactsDir, artifactTarball));
     await mkdirp(dockerBuildDir);
     await unlinkAsync(resolve(dockerBuildDir, artifactTarball));
+    await unlinkAsync(resolve(dockerBuildDir, metricbeatTarball));
+    await unlinkAsync(resolve(dockerBuildDir, filebeatTarball));
   } catch (e) {
     if (e && e.code === 'ENOENT' && e.syscall === 'access') {
       throw new Error(
@@ -108,6 +117,16 @@ export async function runDockerGenerator(
   // Create the kibana linux target inside the
   // Kibana docker build
   await linkAsync(resolve(artifactsDir, artifactTarball), resolve(dockerBuildDir, artifactTarball));
+  if (flags.cloud) {
+    await linkAsync(
+      resolve(artifactsDir, metricbeatTarball),
+      resolve(dockerBuildDir, metricbeatTarball)
+    );
+    await linkAsync(
+      resolve(artifactsDir, filebeatTarball),
+      resolve(dockerBuildDir, filebeatTarball)
+    );
+  }
 
   // Write all the needed docker config files
   // into kibana-docker folder

diff --git a/src/dev/build/tasks/os_packages/docker_generator/template_context.ts b/src/dev/build/tasks/os_packages/docker_generator/template_context.ts
@@ -21,6 +21,9 @@ export interface TemplateContext {
   dockerBuildDate: string;
   usePublicArtifact?: boolean;
   ubi?: boolean;
+  cloud?: boolean;
+  metricbeatTarball?: string;
+  filebeatTarball?: string;
   ironbank?: boolean;
   revision: string;
   architecture?: string;

diff --git a/src/dev/build/tasks/os_packages/docker_generator/templates/base/Dockerfile b/src/dev/build/tasks/os_packages/docker_generator/templates/base/Dockerfile
@@ -36,6 +36,15 @@ RUN tar --strip-components=1 -zxf /opt/kibana.tar.gz
 # REF: https://docs.openshift.org/latest/creating_images/guidelines.html
 RUN chmod -R g=u /usr/share/kibana
 
+{{#cloud}}
+COPY {{filebeatTarball}} /tmp/filebeat.tar.gz
+COPY {{metricbeatTarball}} /tmp/metricbeat.tar.gz
+
+RUN mkdir -p /opt/filebeat /opt/metricbeat && \
+  tar xf /tmp/filebeat.tar.gz -C /opt/filebeat --strip-components=1 && \
+  tar xf /tmp/metricbeat.tar.gz -C /opt/metricbeat --strip-components=1
+{{/cloud}}
+
 ################################################################################
 # Build stage 1 (the actual Kibana image):
 #