diff --git a/.gitignore b/.gitignore index 4f77a6e450c4b..32c77b20ef204 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ /.chromium .DS_Store .node_binaries +/.beats .native_modules node_modules !/src/dev/npm/integration_tests/__fixtures__/fixture1/node_modules diff --git a/src/dev/build/args.test.ts b/src/dev/build/args.test.ts index d3ff822f9a3a3..64d89a650e62e 100644 --- a/src/dev/build/args.test.ts +++ b/src/dev/build/args.test.ts @@ -29,6 +29,7 @@ it('build default and oss dist for current platform, without packages, by defaul "createArchives": true, "createDebPackage": false, "createDockerCentOS": false, + "createDockerCloud": false, "createDockerContexts": true, "createDockerUBI": false, "createExamplePlugins": false, @@ -55,6 +56,7 @@ it('builds packages if --all-platforms is passed', () => { "createArchives": true, "createDebPackage": true, "createDockerCentOS": true, + "createDockerCloud": false, "createDockerContexts": true, "createDockerUBI": true, "createExamplePlugins": false, @@ -81,6 +83,7 @@ it('limits packages if --rpm passed with --all-platforms', () => { "createArchives": true, "createDebPackage": false, "createDockerCentOS": false, + "createDockerCloud": false, "createDockerContexts": true, "createDockerUBI": false, "createExamplePlugins": false, @@ -107,6 +110,7 @@ it('limits packages if --deb passed with --all-platforms', () => { "createArchives": true, "createDebPackage": true, "createDockerCentOS": false, + "createDockerCloud": false, "createDockerContexts": true, "createDockerUBI": false, "createExamplePlugins": false, @@ -134,6 +138,7 @@ it('limits packages if --docker passed with --all-platforms', () => { "createArchives": true, "createDebPackage": false, "createDockerCentOS": true, + "createDockerCloud": false, "createDockerContexts": true, "createDockerUBI": true, "createExamplePlugins": false, @@ -168,6 +173,7 @@ it('limits packages if --docker passed with --skip-docker-ubi and --all-platform "createArchives": true, "createDebPackage": false, "createDockerCentOS": true, + "createDockerCloud": false, "createDockerContexts": true, "createDockerUBI": false, "createExamplePlugins": false, @@ -195,6 +201,7 @@ it('limits packages if --all-platforms passed with --skip-docker-centos', () => "createArchives": true, "createDebPackage": true, "createDockerCentOS": false, + "createDockerCloud": false, "createDockerContexts": true, "createDockerUBI": true, "createExamplePlugins": false, diff --git a/src/dev/build/args.ts b/src/dev/build/args.ts index 9ee375e33d38f..1124d90be89c6 100644 --- a/src/dev/build/args.ts +++ b/src/dev/build/args.ts @@ -26,6 +26,7 @@ export function readCliArgs(argv: string[]) { 'skip-docker-contexts', 'skip-docker-ubi', 'skip-docker-centos', + 'docker-cloud', 'release', 'skip-node-download', 'verbose', @@ -103,6 +104,7 @@ export function readCliArgs(argv: string[]) { createDebPackage: isOsPackageDesired('deb'), createDockerCentOS: isOsPackageDesired('docker-images') && !Boolean(flags['skip-docker-centos']), + createDockerCloud: isOsPackageDesired('docker-images') && Boolean(flags['docker-cloud']), createDockerUBI: isOsPackageDesired('docker-images') && !Boolean(flags['skip-docker-ubi']), createDockerContexts: !Boolean(flags['skip-docker-contexts']), targetAllPlatforms: Boolean(flags['all-platforms']), diff --git a/src/dev/build/build_distributables.ts b/src/dev/build/build_distributables.ts index 1042cdc484c12..39a62c1fd35dc 100644 --- a/src/dev/build/build_distributables.ts +++ b/src/dev/build/build_distributables.ts @@ -22,6 +22,7 @@ export interface BuildOptions { createDebPackage: boolean; createDockerUBI: boolean; createDockerCentOS: boolean; + createDockerCloud: boolean; createDockerContexts: boolean; versionQualifier: string | undefined; targetAllPlatforms: boolean; @@ -127,6 +128,11 @@ export async function buildDistributables(log: ToolingLog, options: BuildOptions await run(Tasks.CreateDockerCentOS); } + if (options.createDockerCloud) { + // control w/ --docker-images and --docker-cloud + await run(Tasks.CreateDockerCloud); + } + if (options.createDockerContexts) { // control w/ --skip-docker-contexts await run(Tasks.CreateDockerContexts); diff --git a/src/dev/build/tasks/os_packages/create_os_package_tasks.ts b/src/dev/build/tasks/os_packages/create_os_package_tasks.ts index 67a9e86ee2073..ab9a7ce65cbc6 100644 --- a/src/dev/build/tasks/os_packages/create_os_package_tasks.ts +++ b/src/dev/build/tasks/os_packages/create_os_package_tasks.ts @@ -91,6 +91,25 @@ export const CreateDockerUBI: Task = { }, }; +export const CreateDockerCloud: Task = { + description: 'Creating Docker Cloud image', + + async run(config, log, build) { + await runDockerGenerator(config, log, build, { + architecture: 'x64', + context: false, + cloud: true, + image: true, + }); + await runDockerGenerator(config, log, build, { + architecture: 'aarch64', + context: false, + cloud: true, + image: true, + }); + }, +}; + export const CreateDockerContexts: Task = { description: 'Creating Docker build contexts', @@ -111,5 +130,10 @@ export const CreateDockerContexts: Task = { context: true, image: false, }); + await runDockerGenerator(config, log, build, { + cloud: true, + context: true, + image: false, + }); }, }; diff --git a/src/dev/build/tasks/os_packages/docker_generator/run.ts b/src/dev/build/tasks/os_packages/docker_generator/run.ts index cac02cae20c42..c5a4ff64d2188 100644 --- a/src/dev/build/tasks/os_packages/docker_generator/run.ts +++ b/src/dev/build/tasks/os_packages/docker_generator/run.ts @@ -7,7 +7,7 @@ */ import { access, link, unlink, chmod } from 'fs'; -import { resolve } from 'path'; +import { resolve, basename } from 'path'; import { promisify } from 'util'; import { ToolingLog, kibanaPackageJson } from '@kbn/dev-utils'; @@ -32,6 +32,7 @@ export async function runDockerGenerator( image: boolean; ubi?: boolean; ironbank?: boolean; + cloud?: boolean; dockerBuildDate?: string; } ) { @@ -42,6 +43,7 @@ export async function runDockerGenerator( let imageFlavor = ''; if (flags.ubi) imageFlavor += `-${ubiVersionTag}`; if (flags.ironbank) imageFlavor += '-ironbank'; + if (flags.cloud) imageFlavor += '-cloud'; // General docker var config const license = 'Elastic License'; @@ -50,7 +52,10 @@ export async function runDockerGenerator( const artifactArchitecture = flags.architecture === 'aarch64' ? 'aarch64' : 'x86_64'; const artifactPrefix = `kibana-${version}-linux`; const artifactTarball = `${artifactPrefix}-${artifactArchitecture}.tar.gz`; + const metricbeatTarball = `metricbeat-${version}-linux-${artifactArchitecture}.tar.gz`; + const filebeatTarball = `filebeat-${version}-linux-${artifactArchitecture}.tar.gz`; const artifactsDir = config.resolveFromTarget('.'); + const beatsDir = config.resolveFromRepo('.beats'); const dockerBuildDate = flags.dockerBuildDate || new Date().toISOString(); // That would produce oss, default and default-ubi7 const dockerBuildDir = config.resolveFromRepo('build', 'kibana-docker', `default${imageFlavor}`); @@ -58,6 +63,13 @@ export async function runDockerGenerator( const dockerTargetFilename = config.resolveFromTarget( `kibana${imageFlavor}-${version}-docker-image${imageArchitecture}.tar.gz` ); + const dependencies = [ + resolve(artifactsDir, artifactTarball), + ...(flags.cloud + ? [resolve(beatsDir, metricbeatTarball), resolve(beatsDir, filebeatTarball)] + : []), + ]; + const scope: TemplateContext = { artifactPrefix, artifactTarball, @@ -72,6 +84,9 @@ export async function runDockerGenerator( baseOSImage, dockerBuildDate, ubi: flags.ubi, + cloud: flags.cloud, + metricbeatTarball, + filebeatTarball, ironbank: flags.ironbank, architecture: flags.architecture, revision: config.getBuildSha(), @@ -87,26 +102,8 @@ export async function runDockerGenerator( return; } - // Verify if we have the needed kibana target in order - // to build the kibana docker image. - // Also create the docker build target folder - // and delete the current linked target into the - // kibana docker build folder if we have one. - try { - await accessAsync(resolve(artifactsDir, artifactTarball)); - await mkdirp(dockerBuildDir); - await unlinkAsync(resolve(dockerBuildDir, artifactTarball)); - } catch (e) { - if (e && e.code === 'ENOENT' && e.syscall === 'access') { - throw new Error( - `Kibana linux target (${artifactTarball}) is needed in order to build ${''}the docker image. None was found at ${artifactsDir}` - ); - } - } - - // Create the kibana linux target inside the - // Kibana docker build - await linkAsync(resolve(artifactsDir, artifactTarball), resolve(dockerBuildDir, artifactTarball)); + // Create the docker build target folder + await mkdirp(dockerBuildDir); // Write all the needed docker config files // into kibana-docker folder @@ -137,6 +134,21 @@ export async function runDockerGenerator( // Only build images on native targets if (flags.image) { + // Link dependencies + for (const src of dependencies) { + const file = basename(src); + const dest = resolve(dockerBuildDir, file); + try { + await accessAsync(src); + await unlinkAsync(dest); + } catch (e) { + if (e && e.code === 'ENOENT' && e.syscall === 'access') { + throw new Error(`${src} is needed in order to build the docker image.`); + } + } + await linkAsync(src, dest); + } + await exec(log, `./build_docker.sh`, [], { cwd: dockerBuildDir, level: 'info', diff --git a/src/dev/build/tasks/os_packages/docker_generator/template_context.ts b/src/dev/build/tasks/os_packages/docker_generator/template_context.ts index 9c9949c9f57ea..075a3a8808e73 100644 --- a/src/dev/build/tasks/os_packages/docker_generator/template_context.ts +++ b/src/dev/build/tasks/os_packages/docker_generator/template_context.ts @@ -21,6 +21,9 @@ export interface TemplateContext { dockerBuildDate: string; usePublicArtifact?: boolean; ubi?: boolean; + cloud?: boolean; + metricbeatTarball?: string; + filebeatTarball?: string; ironbank?: boolean; revision: string; architecture?: string; diff --git a/src/dev/build/tasks/os_packages/docker_generator/templates/base/Dockerfile b/src/dev/build/tasks/os_packages/docker_generator/templates/base/Dockerfile index 60dabbffc6312..078741a0d0f6c 100644 --- a/src/dev/build/tasks/os_packages/docker_generator/templates/base/Dockerfile +++ b/src/dev/build/tasks/os_packages/docker_generator/templates/base/Dockerfile @@ -24,18 +24,27 @@ RUN cd /opt && \ {{/usePublicArtifact}} {{^usePublicArtifact}} -COPY {{artifactTarball}} /opt/kibana.tar.gz +COPY {{artifactTarball}} /tmp/kibana.tar.gz {{/usePublicArtifact}} RUN mkdir /usr/share/kibana WORKDIR /usr/share/kibana -RUN tar --strip-components=1 -zxf /opt/kibana.tar.gz +RUN tar --strip-components=1 -zxf /tmp/kibana.tar.gz # Ensure that group permissions are the same as user permissions. # This will help when relying on GID-0 to run Kibana, rather than UID-1000. # OpenShift does this, for example. # REF: https://docs.openshift.org/latest/creating_images/guidelines.html RUN chmod -R g=u /usr/share/kibana +{{#cloud}} +COPY {{filebeatTarball}} /tmp/filebeat.tar.gz +COPY {{metricbeatTarball}} /tmp/metricbeat.tar.gz + +RUN mkdir -p /opt/filebeat /opt/metricbeat && \ + tar xf /tmp/filebeat.tar.gz -C /opt/filebeat --strip-components=1 && \ + tar xf /tmp/metricbeat.tar.gz -C /opt/metricbeat --strip-components=1 +{{/cloud}} + ################################################################################ # Build stage 1 (the actual Kibana image): # @@ -86,6 +95,9 @@ RUN fc-cache -v # Bring in Kibana from the initial stage. COPY --from=builder --chown=1000:0 /usr/share/kibana /usr/share/kibana +{{#cloud}} +COPY --from=builder --chown=0:0 /opt /opt +{{/cloud}} WORKDIR /usr/share/kibana RUN ln -s /usr/share/kibana /opt/kibana @@ -146,8 +158,19 @@ RUN mkdir /licenses && \ cp LICENSE.txt /licenses/LICENSE {{/ubi}} -USER kibana - ENTRYPOINT ["/bin/tini", "--"] +{{#cloud}} +CMD ["/app/kibana.sh"] +# Generate a stub command that will be overwritten at runtime +RUN mkdir /app && \ + echo -e '#!/bin/sh\nexec /usr/local/bin/kibana-docker' > /app/kibana.sh && \ + chmod 0555 /app/kibana.sh +{{/cloud}} + +{{^cloud}} CMD ["/usr/local/bin/kibana-docker"] +{{/cloud}} + + +USER kibana