From 82a1ff2d3b3d9077b52e4cd2a15af01613a112cf Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Mon, 4 Apr 2022 10:51:33 -0500 Subject: [PATCH] [build] Add flag for using a local Kibana artifact in the Docker context Currently Docker contexts always install a Kibana distribution from an http endpoint. These endpoints are only available after a release has been published, making pre-release testing difficult. This adds a flag to use a local Kibana artifact, with the intention of verifying the Docker contexts with release artifacts prior to publishing the release. --- src/dev/build/args.test.ts | 7 ++++ src/dev/build/args.ts | 3 ++ src/dev/build/build_distributables.ts | 1 + src/dev/build/cli.ts | 33 ++++++++++--------- src/dev/build/lib/config.ts | 11 +++++++ .../nodejs/extract_node_builds_task.test.ts | 1 + .../verify_existing_node_builds_task.test.ts | 1 + .../docker_generator/bundle_dockerfiles.ts | 4 ++- 8 files changed, 44 insertions(+), 17 deletions(-) diff --git a/src/dev/build/args.test.ts b/src/dev/build/args.test.ts index b0f39840ba440..d01a73f08395b 100644 --- a/src/dev/build/args.test.ts +++ b/src/dev/build/args.test.ts @@ -36,6 +36,7 @@ it('build default and oss dist for current platform, without packages, by defaul "createGenericFolders": true, "createPlatformFolders": true, "createRpmPackage": false, + "dockerContextUseLocalArtifact": null, "dockerCrossCompile": false, "dockerPush": false, "dockerTagQualifier": null, @@ -68,6 +69,7 @@ it('builds packages if --all-platforms is passed', () => { "createGenericFolders": true, "createPlatformFolders": true, "createRpmPackage": true, + "dockerContextUseLocalArtifact": null, "dockerCrossCompile": false, "dockerPush": false, "dockerTagQualifier": null, @@ -100,6 +102,7 @@ it('limits packages if --rpm passed with --all-platforms', () => { "createGenericFolders": true, "createPlatformFolders": true, "createRpmPackage": true, + "dockerContextUseLocalArtifact": null, "dockerCrossCompile": false, "dockerPush": false, "dockerTagQualifier": null, @@ -132,6 +135,7 @@ it('limits packages if --deb passed with --all-platforms', () => { "createGenericFolders": true, "createPlatformFolders": true, "createRpmPackage": false, + "dockerContextUseLocalArtifact": null, "dockerCrossCompile": false, "dockerPush": false, "dockerTagQualifier": null, @@ -165,6 +169,7 @@ it('limits packages if --docker passed with --all-platforms', () => { "createGenericFolders": true, "createPlatformFolders": true, "createRpmPackage": false, + "dockerContextUseLocalArtifact": null, "dockerCrossCompile": false, "dockerPush": false, "dockerTagQualifier": null, @@ -205,6 +210,7 @@ it('limits packages if --docker passed with --skip-docker-ubi and --all-platform "createGenericFolders": true, "createPlatformFolders": true, "createRpmPackage": false, + "dockerContextUseLocalArtifact": null, "dockerCrossCompile": false, "dockerPush": false, "dockerTagQualifier": null, @@ -238,6 +244,7 @@ it('limits packages if --all-platforms passed with --skip-docker-ubuntu', () => "createGenericFolders": true, "createPlatformFolders": true, "createRpmPackage": true, + "dockerContextUseLocalArtifact": null, "dockerCrossCompile": false, "dockerPush": false, "dockerTagQualifier": null, diff --git a/src/dev/build/args.ts b/src/dev/build/args.ts index 2bad2c0721e2e..d1b094e97fdc5 100644 --- a/src/dev/build/args.ts +++ b/src/dev/build/args.ts @@ -22,6 +22,7 @@ export function readCliArgs(argv: string[]) { 'skip-os-packages', 'rpm', 'deb', + 'docker-context-use-local-artifact', 'docker-cross-compile', 'docker-images', 'docker-push', @@ -53,6 +54,7 @@ export function readCliArgs(argv: string[]) { rpm: null, deb: null, 'docker-images': null, + 'docker-context-use-local-artifact': null, 'docker-cross-compile': false, 'docker-push': false, 'docker-tag-qualifier': null, @@ -114,6 +116,7 @@ export function readCliArgs(argv: string[]) { const buildOptions: BuildOptions = { isRelease: Boolean(flags.release), versionQualifier: flags['version-qualifier'], + dockerContextUseLocalArtifact: flags['docker-context-use-local-artifact'], dockerCrossCompile: Boolean(flags['docker-cross-compile']), dockerPush: Boolean(flags['docker-push']), dockerTagQualifier: flags['docker-tag-qualifier'], diff --git a/src/dev/build/build_distributables.ts b/src/dev/build/build_distributables.ts index d2b2d24667bce..f8df9ec61609d 100644 --- a/src/dev/build/build_distributables.ts +++ b/src/dev/build/build_distributables.ts @@ -13,6 +13,7 @@ import * as Tasks from './tasks'; export interface BuildOptions { isRelease: boolean; + dockerContextUseLocalArtifact: boolean | null; dockerCrossCompile: boolean; dockerPush: boolean; dockerTagQualifier: string | null; diff --git a/src/dev/build/cli.ts b/src/dev/build/cli.ts index 561e2aea5c15d..5acdc4b37423f 100644 --- a/src/dev/build/cli.ts +++ b/src/dev/build/cli.ts @@ -33,22 +33,23 @@ if (showHelp) { build the Kibana distributable options: - --skip-archives {dim Don't produce tar/zip archives} - --skip-os-packages {dim Don't produce rpm/deb/docker packages} - --all-platforms {dim Produce archives for all platforms, not just this one} - --rpm {dim Only build the rpm packages} - --deb {dim Only build the deb packages} - --docker-images {dim Only build the Docker images} - --docker-cross-compile {dim Produce arm64 and amd64 Docker images} - --docker-contexts {dim Only build the Docker build contexts} - --skip-docker-ubi {dim Don't build the docker ubi image} - --skip-docker-ubuntu {dim Don't build the docker ubuntu image} - --release {dim Produce a release-ready distributable} - --version-qualifier {dim Suffix version with a qualifier} - --skip-node-download {dim Reuse existing downloads of node.js} - --verbose,-v {dim Turn on verbose logging} - --no-debug {dim Turn off debug logging} - --epr-registry {dim Specify the EPR registry to use for Fleet packages, 'production' or 'snapshot'} + --skip-archives {dim Don't produce tar/zip archives} + --skip-os-packages {dim Don't produce rpm/deb/docker packages} + --all-platforms {dim Produce archives for all platforms, not just this one} + --rpm {dim Only build the rpm packages} + --deb {dim Only build the deb packages} + --docker-images {dim Only build the Docker images} + --docker-context-use-local-artifact {dim Use a local artifact when building the Docker context} + --docker-cross-compile {dim Produce arm64 and amd64 Docker images} + --docker-contexts {dim Only build the Docker build contexts} + --skip-docker-ubi {dim Don't build the docker ubi image} + --skip-docker-ubuntu {dim Don't build the docker ubuntu image} + --release {dim Produce a release-ready distributable} + --version-qualifier {dim Suffix version with a qualifier} + --skip-node-download {dim Reuse existing downloads of node.js} + --verbose,-v {dim Turn on verbose logging} + --no-debug {dim Turn off debug logging} + --epr-registry {dim Specify the EPR registry to use for Fleet packages, 'production' or 'snapshot'} `) + '\n' ); process.exit(1); diff --git a/src/dev/build/lib/config.ts b/src/dev/build/lib/config.ts index 2bab1d28f9ef7..bcd294f0d7f4a 100644 --- a/src/dev/build/lib/config.ts +++ b/src/dev/build/lib/config.ts @@ -17,6 +17,7 @@ interface Options { isRelease: boolean; targetAllPlatforms: boolean; versionQualifier?: string; + dockerContextUseLocalArtifact: boolean | null; dockerCrossCompile: boolean; dockerTagQualifier: string | null; dockerPush: boolean; @@ -36,6 +37,7 @@ export class Config { isRelease, targetAllPlatforms, versionQualifier, + dockerContextUseLocalArtifact, dockerCrossCompile, dockerTagQualifier, dockerPush, @@ -53,6 +55,7 @@ export class Config { versionQualifier, pkg, }), + dockerContextUseLocalArtifact, dockerCrossCompile, dockerTagQualifier, dockerPush, @@ -66,6 +69,7 @@ export class Config { private readonly nodeVersion: string, private readonly repoRoot: string, private readonly versionInfo: VersionInfo, + private readonly dockerContextUseLocalArtifact: boolean | null, private readonly dockerCrossCompile: boolean, private readonly dockerTagQualifier: string | null, private readonly dockerPush: boolean, @@ -100,6 +104,13 @@ export class Config { return this.dockerPush; } + /** + * Use a local Kibana distribution when producing a docker context + */ + getDockerContextUseLocalArtifact() { + return this.dockerContextUseLocalArtifact; + } + /** * Get docker cross compile */ diff --git a/src/dev/build/tasks/nodejs/extract_node_builds_task.test.ts b/src/dev/build/tasks/nodejs/extract_node_builds_task.test.ts index 0041829984aa7..b8992bf6e590b 100644 --- a/src/dev/build/tasks/nodejs/extract_node_builds_task.test.ts +++ b/src/dev/build/tasks/nodejs/extract_node_builds_task.test.ts @@ -43,6 +43,7 @@ async function setup() { const config = await Config.create({ isRelease: true, targetAllPlatforms: true, + dockerContextUseLocalArtifact: false, dockerCrossCompile: false, dockerPush: false, dockerTagQualifier: '', diff --git a/src/dev/build/tasks/nodejs/verify_existing_node_builds_task.test.ts b/src/dev/build/tasks/nodejs/verify_existing_node_builds_task.test.ts index 85458c29ddcff..96d8b7cc6336f 100644 --- a/src/dev/build/tasks/nodejs/verify_existing_node_builds_task.test.ts +++ b/src/dev/build/tasks/nodejs/verify_existing_node_builds_task.test.ts @@ -48,6 +48,7 @@ async function setup(actualShaSums?: Record) { const config = await Config.create({ isRelease: true, targetAllPlatforms: true, + dockerContextUseLocalArtifact: false, dockerCrossCompile: false, dockerPush: false, dockerTagQualifier: '', diff --git a/src/dev/build/tasks/os_packages/docker_generator/bundle_dockerfiles.ts b/src/dev/build/tasks/os_packages/docker_generator/bundle_dockerfiles.ts index cc1ffb5f3e301..9dfff321fc7a2 100644 --- a/src/dev/build/tasks/os_packages/docker_generator/bundle_dockerfiles.ts +++ b/src/dev/build/tasks/os_packages/docker_generator/bundle_dockerfiles.ts @@ -23,6 +23,7 @@ export async function bundleDockerFiles(config: Config, log: ToolingLog, scope: const dockerFilesDirName = `kibana${scope.imageFlavor}-${scope.version}-docker-build-context`; const dockerFilesBuildDir = resolve(scope.dockerBuildDir, dockerFilesDirName); const dockerFilesOutputDir = config.resolveFromTarget(`${dockerFilesDirName}.tar.gz`); + const dockerContextUseLocalArtifact = config.getDockerContextUseLocalArtifact(); // Create dockerfiles dir inside docker build dir await mkdirp(dockerFilesBuildDir); @@ -32,7 +33,8 @@ export async function bundleDockerFiles(config: Config, log: ToolingLog, scope: resolve(dockerFilesBuildDir, dockerfileTemplate.name), dockerfileTemplate.generator({ ...scope, - usePublicArtifact: true, + usePublicArtifact: + dockerContextUseLocalArtifact !== null ? !dockerContextUseLocalArtifact : true, }) );