From ace722085fd5070b72801a53ba4c5810d4a51c4f Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Mon, 9 Aug 2021 09:22:26 -0500 Subject: [PATCH] [build] Add cloud docker images This adds a new build task to create cloud docker images. These images include metricbeat and filebeat. Archives are for both are expected to be in the target folder prior to starting the build. The cloud image can be generated with `node scripts/build --all-platforms --docker-cloud` Closes #99916 --- src/dev/build/args.test.ts | 7 +++++++ src/dev/build/args.ts | 2 ++ src/dev/build/build_distributables.ts | 6 ++++++ .../os_packages/create_os_package_tasks.ts | 19 +++++++++++++++++++ .../tasks/os_packages/docker_generator/run.ts | 19 +++++++++++++++++++ .../docker_generator/template_context.ts | 3 +++ .../templates/base/Dockerfile | 9 +++++++++ 7 files changed, 65 insertions(+) diff --git a/src/dev/build/args.test.ts b/src/dev/build/args.test.ts index d3ff822f9a3a3..64d89a650e62e 100644 --- a/src/dev/build/args.test.ts +++ b/src/dev/build/args.test.ts @@ -29,6 +29,7 @@ it('build default and oss dist for current platform, without packages, by defaul "createArchives": true, "createDebPackage": false, "createDockerCentOS": false, + "createDockerCloud": false, "createDockerContexts": true, "createDockerUBI": false, "createExamplePlugins": false, @@ -55,6 +56,7 @@ it('builds packages if --all-platforms is passed', () => { "createArchives": true, "createDebPackage": true, "createDockerCentOS": true, + "createDockerCloud": false, "createDockerContexts": true, "createDockerUBI": true, "createExamplePlugins": false, @@ -81,6 +83,7 @@ it('limits packages if --rpm passed with --all-platforms', () => { "createArchives": true, "createDebPackage": false, "createDockerCentOS": false, + "createDockerCloud": false, "createDockerContexts": true, "createDockerUBI": false, "createExamplePlugins": false, @@ -107,6 +110,7 @@ it('limits packages if --deb passed with --all-platforms', () => { "createArchives": true, "createDebPackage": true, "createDockerCentOS": false, + "createDockerCloud": false, "createDockerContexts": true, "createDockerUBI": false, "createExamplePlugins": false, @@ -134,6 +138,7 @@ it('limits packages if --docker passed with --all-platforms', () => { "createArchives": true, "createDebPackage": false, "createDockerCentOS": true, + "createDockerCloud": false, "createDockerContexts": true, "createDockerUBI": true, "createExamplePlugins": false, @@ -168,6 +173,7 @@ it('limits packages if --docker passed with --skip-docker-ubi and --all-platform "createArchives": true, "createDebPackage": false, "createDockerCentOS": true, + "createDockerCloud": false, "createDockerContexts": true, "createDockerUBI": false, "createExamplePlugins": false, @@ -195,6 +201,7 @@ it('limits packages if --all-platforms passed with --skip-docker-centos', () => "createArchives": true, "createDebPackage": true, "createDockerCentOS": false, + "createDockerCloud": false, "createDockerContexts": true, "createDockerUBI": true, "createExamplePlugins": false, diff --git a/src/dev/build/args.ts b/src/dev/build/args.ts index 9ee375e33d38f..1124d90be89c6 100644 --- a/src/dev/build/args.ts +++ b/src/dev/build/args.ts @@ -26,6 +26,7 @@ export function readCliArgs(argv: string[]) { 'skip-docker-contexts', 'skip-docker-ubi', 'skip-docker-centos', + 'docker-cloud', 'release', 'skip-node-download', 'verbose', @@ -103,6 +104,7 @@ export function readCliArgs(argv: string[]) { createDebPackage: isOsPackageDesired('deb'), createDockerCentOS: isOsPackageDesired('docker-images') && !Boolean(flags['skip-docker-centos']), + createDockerCloud: isOsPackageDesired('docker-images') && Boolean(flags['docker-cloud']), createDockerUBI: isOsPackageDesired('docker-images') && !Boolean(flags['skip-docker-ubi']), createDockerContexts: !Boolean(flags['skip-docker-contexts']), targetAllPlatforms: Boolean(flags['all-platforms']), diff --git a/src/dev/build/build_distributables.ts b/src/dev/build/build_distributables.ts index 9ddf02e101a19..315f9270e785e 100644 --- a/src/dev/build/build_distributables.ts +++ b/src/dev/build/build_distributables.ts @@ -22,6 +22,7 @@ export interface BuildOptions { createDebPackage: boolean; createDockerUBI: boolean; createDockerCentOS: boolean; + createDockerCloud: boolean; createDockerContexts: boolean; versionQualifier: string | undefined; targetAllPlatforms: boolean; @@ -123,6 +124,11 @@ export async function buildDistributables(log: ToolingLog, options: BuildOptions await run(Tasks.CreateDockerCentOS); } + if (options.createDockerCloud) { + // control w/ --docker-images and --docker-cloud + await run(Tasks.CreateDockerCloud); + } + if (options.createDockerContexts) { // control w/ --skip-docker-contexts await run(Tasks.CreateDockerContexts); diff --git a/src/dev/build/tasks/os_packages/create_os_package_tasks.ts b/src/dev/build/tasks/os_packages/create_os_package_tasks.ts index 99d0e1998e78a..ced8a9622725c 100644 --- a/src/dev/build/tasks/os_packages/create_os_package_tasks.ts +++ b/src/dev/build/tasks/os_packages/create_os_package_tasks.ts @@ -82,6 +82,25 @@ export const CreateDockerUBI: Task = { }, }; +export const CreateDockerCloud: Task = { + description: 'Creating Docker Cloud image', + + async run(config, log, build) { + await runDockerGenerator(config, log, build, { + architecture: 'x64', + context: false, + cloud: true, + image: true, + }); + await runDockerGenerator(config, log, build, { + architecture: 'aarch64', + context: false, + cloud: true, + image: true, + }); + }, +}; + export const CreateDockerContexts: Task = { description: 'Creating Docker build contexts', diff --git a/src/dev/build/tasks/os_packages/docker_generator/run.ts b/src/dev/build/tasks/os_packages/docker_generator/run.ts index 97fd740409741..9542601ed2b3b 100644 --- a/src/dev/build/tasks/os_packages/docker_generator/run.ts +++ b/src/dev/build/tasks/os_packages/docker_generator/run.ts @@ -33,6 +33,7 @@ export async function runDockerGenerator( image: boolean; ubi?: boolean; ironbank?: boolean; + cloud?: boolean; dockerBuildDate?: string; } ) { @@ -43,6 +44,7 @@ export async function runDockerGenerator( let imageFlavor = ''; if (flags.ubi) imageFlavor += `-${ubiVersionTag}`; if (flags.ironbank) imageFlavor += '-ironbank'; + if (flags.cloud) imageFlavor += '-cloud'; // General docker var config const license = 'Elastic License'; @@ -51,6 +53,8 @@ export async function runDockerGenerator( const artifactArchitecture = flags.architecture === 'aarch64' ? 'aarch64' : 'x86_64'; const artifactPrefix = `kibana-${version}-linux`; const artifactTarball = `${artifactPrefix}-${artifactArchitecture}.tar.gz`; + const metricbeatTarball = `metricbeat-${version}-linux-${artifactArchitecture}.tar.gz`; + const filebeatTarball = `filebeat-${version}-linux-${artifactArchitecture}.tar.gz`; const artifactsDir = config.resolveFromTarget('.'); const dockerBuildDate = flags.dockerBuildDate || new Date().toISOString(); // That would produce oss, default and default-ubi7 @@ -73,6 +77,9 @@ export async function runDockerGenerator( baseOSImage, dockerBuildDate, ubi: flags.ubi, + cloud: flags.cloud, + metricbeatTarball, + filebeatTarball, ironbank: flags.ironbank, architecture: flags.architecture, revision: config.getBuildSha(), @@ -97,6 +104,8 @@ export async function runDockerGenerator( await accessAsync(resolve(artifactsDir, artifactTarball)); await mkdirp(dockerBuildDir); await unlinkAsync(resolve(dockerBuildDir, artifactTarball)); + await unlinkAsync(resolve(dockerBuildDir, metricbeatTarball)); + await unlinkAsync(resolve(dockerBuildDir, filebeatTarball)); } catch (e) { if (e && e.code === 'ENOENT' && e.syscall === 'access') { throw new Error( @@ -108,6 +117,16 @@ export async function runDockerGenerator( // Create the kibana linux target inside the // Kibana docker build await linkAsync(resolve(artifactsDir, artifactTarball), resolve(dockerBuildDir, artifactTarball)); + if (flags.cloud) { + await linkAsync( + resolve(artifactsDir, metricbeatTarball), + resolve(dockerBuildDir, metricbeatTarball) + ); + await linkAsync( + resolve(artifactsDir, filebeatTarball), + resolve(dockerBuildDir, filebeatTarball) + ); + } // Write all the needed docker config files // into kibana-docker folder diff --git a/src/dev/build/tasks/os_packages/docker_generator/template_context.ts b/src/dev/build/tasks/os_packages/docker_generator/template_context.ts index 9c9949c9f57ea..075a3a8808e73 100644 --- a/src/dev/build/tasks/os_packages/docker_generator/template_context.ts +++ b/src/dev/build/tasks/os_packages/docker_generator/template_context.ts @@ -21,6 +21,9 @@ export interface TemplateContext { dockerBuildDate: string; usePublicArtifact?: boolean; ubi?: boolean; + cloud?: boolean; + metricbeatTarball?: string; + filebeatTarball?: string; ironbank?: boolean; revision: string; architecture?: string; diff --git a/src/dev/build/tasks/os_packages/docker_generator/templates/base/Dockerfile b/src/dev/build/tasks/os_packages/docker_generator/templates/base/Dockerfile index eb4708b6ac555..29fe8239c9057 100644 --- a/src/dev/build/tasks/os_packages/docker_generator/templates/base/Dockerfile +++ b/src/dev/build/tasks/os_packages/docker_generator/templates/base/Dockerfile @@ -36,6 +36,15 @@ RUN tar --strip-components=1 -zxf /opt/kibana.tar.gz # REF: https://docs.openshift.org/latest/creating_images/guidelines.html RUN chmod -R g=u /usr/share/kibana +{{#cloud}} +COPY {{filebeatTarball}} /tmp/filebeat.tar.gz +COPY {{metricbeatTarball}} /tmp/metricbeat.tar.gz + +RUN mkdir -p /opt/filebeat /opt/metricbeat && \ + tar xf /tmp/filebeat.tar.gz -C /opt/filebeat --strip-components=1 && \ + tar xf /tmp/metricbeat.tar.gz -C /opt/metricbeat --strip-components=1 +{{/cloud}} + ################################################################################ # Build stage 1 (the actual Kibana image): #