Testing Elastic-wide Buildkite system

.buildkite/pipeline-utils/buildkite/client.ts

@@ -29,7 +29,13 @@ export interface BuildkiteStep {
   label: string;
   parallelism?: number;
   agents: {
-    queue: string;
+    queue?: string;
+    provider?: string;
+    image?: string;
+    imageProject?: string;
+    machineType?: string;
+    minCpuPlatform?: string;
+    preemptible?: boolean;
   };
   timeout_in_minutes?: number;
   key?: string;

.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts

@@ -404,7 +404,11 @@ export async function pickTestGroupRunOrder() {
             timeout_in_minutes: 120,
             key: 'jest',
             agents: {
-              queue: 'n2-4-spot',
+              provider: 'gcp',
+              image: 'family/kibana-ubuntu-2004',
+              imageProject: 'elastic-images-qa',
+              machineType: 'n2-standard-4',
+              preemptible: true,
             },
             retry: {
               automatic: [
@@ -424,7 +428,11 @@ export async function pickTestGroupRunOrder() {
             timeout_in_minutes: 120,
             key: 'jest-integration',
             agents: {
-              queue: 'n2-4-spot',
+              provider: 'gcp',
+              image: 'family/kibana-ubuntu-2004',
+              imageProject: 'elastic-images-qa',
+              machineType: 'n2-standard-4',
+              preemptible: true,
             },
             retry: {
               automatic: [
@@ -455,12 +463,17 @@ export async function pickTestGroupRunOrder() {
                   : -1
               )
               .map(
-                ({ title, key, queue = defaultQueue }): BuildkiteStep => ({
+                // ({ title, key, queue = defaultQueue }): BuildkiteStep => ({
+                ({ title, key }): BuildkiteStep => ({
                   label: title,
                   command: getRequiredEnv('FTR_CONFIGS_SCRIPT'),
                   timeout_in_minutes: 90,
                   agents: {
-                    queue,
+                    provider: 'gcp',
+                    image: 'family/kibana-ubuntu-2004',
+                    imageProject: 'elastic-images-qa',
+                    machineType: 'n2-standard-4',
+                    preemptible: true,
                   },
                   env: {
                     FTR_CONFIG_GROUP_KEY: key,

.buildkite/pipelines/pull_request/base.yml

@@ -3,14 +3,23 @@ steps:
     label: Pre-Build
     timeout_in_minutes: 10
     agents:
-      queue: kibana-default
+      provider: gcp
+      preemptible: true
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      machineType: n2-standard-2
 
   - wait
 
   - command: .buildkite/scripts/steps/build_kibana.sh
     label: Build Kibana Distribution and Plugins
     agents:
-      queue: n2-16-spot
+      provider: gcp
+      preemptible: true
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      machineType: n2-standard-16
+      buildDirectory: /dev/shm/bk
     key: build
     if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
     timeout_in_minutes: 60
@@ -22,7 +31,11 @@ steps:
   - command: .buildkite/scripts/steps/build_api_docs.sh
     label: 'Build API Docs'
     agents:
-      queue: n2-4-spot
+      provider: gcp
+      preemptible: true
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      machineType: n2-standard-4
     key: build_api_docs
     timeout_in_minutes: 60
     retry:
@@ -33,7 +46,11 @@ steps:
   - command: .buildkite/scripts/steps/ci_stats_ready.sh
     label: Mark CI Stats as ready
     agents:
-      queue: kibana-default
+      provider: gcp
+      preemptible: true
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      machineType: n2-standard-2
     timeout_in_minutes: 10
     depends_on:
       - build
@@ -46,7 +63,11 @@ steps:
   - command: .buildkite/scripts/steps/test/pick_test_group_run_order.sh
     label: 'Pick Test Group Run Order'
     agents:
-      queue: kibana-default
+      provider: gcp
+      preemptible: true
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      machineType: n2-standard-2
     env:
       JEST_UNIT_SCRIPT: '.buildkite/scripts/steps/test/jest.sh'
       JEST_INTEGRATION_SCRIPT: '.buildkite/scripts/steps/test/jest_integration.sh'
@@ -59,7 +80,11 @@ steps:
   - command: .buildkite/scripts/steps/lint.sh
     label: 'Linting'
     agents:
-      queue: n2-8-spot
+      provider: gcp
+      preemptible: true
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      machineType: n2-standard-8
     key: linting
     timeout_in_minutes: 60
     retry:
@@ -70,7 +95,12 @@ steps:
   - command: .buildkite/scripts/steps/lint_with_types.sh
     label: 'Linting (with types)'
     agents:
-      queue: n2-16-spot
+      provider: gcp
+      preemptible: true
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      machineType: n2-standard-16
+      buildDirectory: /dev/shm/bk
     key: linting_with_types
     timeout_in_minutes: 90
     retry:
@@ -81,7 +111,12 @@ steps:
   - command: .buildkite/scripts/steps/checks.sh
     label: 'Checks'
     agents:
-      queue: n2-2-spot
+      provider: gcp
+      preemptible: true
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      machineType: n2-standard-2
+      buildDirectory: /dev/shm/bk
     timeout_in_minutes: 60
     retry:
       automatic:
@@ -91,7 +126,12 @@ steps:
   - command: .buildkite/scripts/steps/check_types.sh
     label: 'Check Types'
     agents:
-      queue: n2-16-spot
+      provider: gcp
+      preemptible: true
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      machineType: n2-standard-16
+      buildDirectory: /dev/shm/bk
     timeout_in_minutes: 60
     retry:
       automatic:

.buildkite/pipelines/pull_request/post_build.yml

@@ -5,4 +5,8 @@ steps:
   - command: .buildkite/scripts/lifecycle/post_build.sh
     label: Post-Build
     agents:
-      queue: kibana-default
+      provider: gcp
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      machineType: n2-standard-2
+      preemptible: true

.buildkite/scripts/common/util.sh

@@ -170,3 +170,15 @@ npm_install_global() {
 download_artifact() {
   retry 3 1 timeout 3m buildkite-agent artifact download "$@"
 }
+
+vault_get() {
+  path=$1
+  field=$2
+
+  fullPath="secret/ci/elastic-kibana/migrated/$path"
+  if [[ "$VAULT_ADDR" == *"secrets.elastic.co"* ]]; then
+    fullPath="secret/kibana-issues/dev/$path"
+  fi
+
+  retry 5 5 vault read -field="$field" "$fullPath"
+}

.buildkite/scripts/lifecycle/pre_build.sh

@@ -8,8 +8,8 @@ if [[ "${GITHUB_BUILD_COMMIT_STATUS_ENABLED:-}" != "true" ]]; then
   "$(dirname "${0}")/commit_status_start.sh"
 fi
 
-export CI_STATS_TOKEN="$(retry 5 5 vault read -field=api_token secret/kibana-issues/dev/kibana_ci_stats)"
-export CI_STATS_HOST="$(retry 5 5 vault read -field=api_host secret/kibana-issues/dev/kibana_ci_stats)"
+export CI_STATS_TOKEN="$(vault_get kibana_ci_stats api_token)"
+export CI_STATS_HOST="$(vault_get kibana_ci_stats api_host)"
 
 ts-node "$(dirname "${0}")/ci_stats_start.ts"
 

.buildkite/scripts/lifecycle/pre_command.sh

@@ -8,7 +8,7 @@ echo '--- Setup environment vars'
 source .buildkite/scripts/common/env.sh
 source .buildkite/scripts/common/setup_node.sh
 
-BUILDKITE_TOKEN="$(retry 5 5 vault read -field=buildkite_token_all_jobs secret/kibana-issues/dev/buildkite-ci)"
+BUILDKITE_TOKEN="$(vault_get buildkite-ci buildkite_token_all_jobs)"
 export BUILDKITE_TOKEN
 
 echo '--- Install/build buildkite dependencies'
@@ -81,10 +81,10 @@ EOF
   if [[ "$CI_STATS_BUILD_ID" ]]; then
     echo "CI Stats Build ID: $CI_STATS_BUILD_ID"
 
-    CI_STATS_TOKEN="$(retry 5 5 vault read -field=api_token secret/kibana-issues/dev/kibana_ci_stats)"
+    CI_STATS_TOKEN="$(vault_get kibana_ci_stats api_token)"
     export CI_STATS_TOKEN
 
-    CI_STATS_HOST="$(retry 5 5 vault read -field=api_host secret/kibana-issues/dev/kibana_ci_stats)"
+    CI_STATS_HOST="$(vault_get kibana_ci_stats api_host)"
     export CI_STATS_HOST
 
     KIBANA_CI_STATS_CONFIG=$(jq -n \
@@ -97,57 +97,57 @@ EOF
   fi
 }
 
-GITHUB_TOKEN=$(retry 5 5 vault read -field=github_token secret/kibana-issues/dev/kibanamachine)
+GITHUB_TOKEN=$(vault_get kibanamachine github_token)
 export GITHUB_TOKEN
 
-KIBANA_CI_GITHUB_TOKEN=$(retry 5 5 vault read -field=github_token secret/kibana-issues/dev/kibana-ci-github)
+KIBANA_CI_GITHUB_TOKEN=$(vault_get kibana-ci-github github_token)
 export KIBANA_CI_GITHUB_TOKEN
 
-KIBANA_CI_REPORTER_KEY=$(retry 5 5 vault read -field=value secret/kibana-issues/dev/kibanamachine-reporter)
+KIBANA_CI_REPORTER_KEY=$(vault_get kibanamachine-reporter value)
 export KIBANA_CI_REPORTER_KEY
 
-KIBANA_DOCKER_USERNAME="$(retry 5 5 vault read -field=username secret/kibana-issues/dev/container-registry)"
+KIBANA_DOCKER_USERNAME="$(vault_get container-registry username)"
 export KIBANA_DOCKER_USERNAME
 
-KIBANA_DOCKER_PASSWORD="$(retry 5 5 vault read -field=password secret/kibana-issues/dev/container-registry)"
+KIBANA_DOCKER_PASSWORD="$(vault_get container-registry password)"
 export KIBANA_DOCKER_PASSWORD
 
-EC_API_KEY="$(retry 5 5 vault read -field=pr_deploy_api_key secret/kibana-issues/dev/kibana-ci-cloud-deploy)"
+EC_API_KEY="$(vault_get kibana-ci-cloud-deploy pr_deploy_api_key)"
 export EC_API_KEY
 
-SYNTHETICS_SERVICE_USERNAME="$(retry 5 5 vault read -field=username secret/kibana-issues/dev/kibana-ci-synthetics-credentials)"
+SYNTHETICS_SERVICE_USERNAME="$(vault_get kibana-ci-synthetics-credentials username)"
 export SYNTHETICS_SERVICE_USERNAME
 
-SYNTHETICS_SERVICE_PASSWORD="$(retry 5 5 vault read -field=password secret/kibana-issues/dev/kibana-ci-synthetics-credentials)"
+SYNTHETICS_SERVICE_PASSWORD="$(vault_get kibana-ci-synthetics-credentials password)"
 export SYNTHETICS_SERVICE_PASSWORD
 
-SYNTHETICS_SERVICE_MANIFEST="$(retry 5 5 vault read -field=manifest secret/kibana-issues/dev/kibana-ci-synthetics-credentials)"
+SYNTHETICS_SERVICE_MANIFEST="$(vault_get kibana-ci-synthetics-credentials manifest)"
 export SYNTHETICS_SERVICE_MANIFEST
 
-SYNTHETICS_REMOTE_KIBANA_USERNAME="$(retry 5 5 vault read -field=username secret/kibana-issues/dev/kibana-ci-synthetics-remote-credentials)"
+SYNTHETICS_REMOTE_KIBANA_USERNAME="$(vault_get kibana-ci-synthetics-remote-credentials username)"
 export SYNTHETICS_REMOTE_KIBANA_USERNAME
 
-SYNTHETICS_REMOTE_KIBANA_PASSWORD="$(retry 5 5 vault read -field=password secret/kibana-issues/dev/kibana-ci-synthetics-remote-credentials)"
+SYNTHETICS_REMOTE_KIBANA_PASSWORD="$(vault_get kibana-ci-synthetics-remote-credentials password)"
 export SYNTHETICS_REMOTE_KIBANA_PASSWORD
 
-SYNTHETICS_REMOTE_KIBANA_URL=${SYNTHETICS_REMOTE_KIBANA_URL-"$(retry 5 5 vault read -field=url secret/kibana-issues/dev/kibana-ci-synthetics-remote-credentials)"}
+SYNTHETICS_REMOTE_KIBANA_URL=${SYNTHETICS_REMOTE_KIBANA_URL-"$(vault_get kibana-ci-synthetics-remote-credentials url)"}
 export SYNTHETICS_REMOTE_KIBANA_URL
 
 # Setup Failed Test Reporter Elasticsearch credentials
 {
-  TEST_FAILURES_ES_CLOUD_ID=$(retry 5 5 vault read -field=cloud_id secret/kibana-issues/dev/failed_tests_reporter_es)
+  TEST_FAILURES_ES_CLOUD_ID=$(vault_get failed_tests_reporter_es cloud_id)
   export TEST_FAILURES_ES_CLOUD_ID
 
-  TEST_FAILURES_ES_USERNAME=$(retry 5 5 vault read -field=username secret/kibana-issues/dev/failed_tests_reporter_es)
+  TEST_FAILURES_ES_USERNAME=$(vault_get failed_tests_reporter_es username)
   export TEST_FAILURES_ES_USERNAME
 
-  TEST_FAILURES_ES_PASSWORD=$(retry 5 5 vault read -field=password secret/kibana-issues/dev/failed_tests_reporter_es)
+  TEST_FAILURES_ES_PASSWORD=$(vault_get failed_tests_reporter_es password)
   export TEST_FAILURES_ES_PASSWORD
 }
 
 BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE="$HOME/.kibana-ci-bazel-remote-cache-local-dev.json"
 export BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE
-retry 5 5 vault read -field=service_account_json secret/kibana-issues/dev/kibana-ci-bazel-remote-cache-local-dev > "$BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE"
+vault_get kibana-ci-bazel-remote-cache-local-dev service_account_json > "$BAZEL_LOCAL_DEV_CACHE_CREDENTIALS_FILE"
 
 PIPELINE_PRE_COMMAND=${PIPELINE_PRE_COMMAND:-".buildkite/scripts/lifecycle/pipelines/$BUILDKITE_PIPELINE_SLUG/pre_command.sh"}
 if [[ -f "$PIPELINE_PRE_COMMAND" ]]; then

.buildkite/scripts/pipelines/pull_request/pipeline.ts

@@ -26,15 +26,15 @@ const getPipeline = (filename: string, removeSteps = true) => {
   return removeSteps ? str.replace(/^steps:/, '') : str;
 };
 
-const uploadPipeline = (pipelineContent: string | object) => {
-  const str =
-    typeof pipelineContent === 'string' ? pipelineContent : JSON.stringify(pipelineContent);
-
-  execSync('buildkite-agent pipeline upload', {
-    input: str,
-    stdio: ['pipe', 'inherit', 'inherit'],
-  });
-};
+// const uploadPipeline = (pipelineContent: string | object) => {
+//   const str =
+//     typeof pipelineContent === 'string' ? pipelineContent : JSON.stringify(pipelineContent);
+
+//   execSync('buildkite-agent pipeline upload', {
+//     input: str,
+//     stdio: ['pipe', 'inherit', 'inherit'],
+//   });
+// };
 
 (async () => {
   try {
@@ -189,7 +189,7 @@ const uploadPipeline = (pipelineContent: string | object) => {
 
     pipeline.push(getPipeline('.buildkite/pipelines/pull_request/post_build.yml'));
 
-    uploadPipeline(pipeline.join('\n'));
+    console.log(pipeline.join('\n'));
   } catch (ex) {
     console.error('PR pipeline generation error', ex.message);
     process.exit(1);