[Ops] POC: PR-build on CI-Systems buildkite (cont'd)

.buildkite/hooks/post-command

@@ -1,7 +1,3 @@
 #!/usr/bin/env bash
 
-if [[ "$BUILDKITE_AGENT_NAME" =~ ^bk-agent ]]; then
-  echo "Pipeline file triggered from outside the kibana executors, skipping post_command"
-else
-  .buildkite/scripts/lifecycle/post_command.sh
-fi
+.buildkite/scripts/lifecycle/post_command.sh

.buildkite/hooks/pre-command

@@ -1,8 +1,5 @@
 #!/usr/bin/env bash
 
-if [[ "$BUILDKITE_AGENT_NAME" =~ ^bk-agent ]]; then
-  echo "Pipeline file triggered from outside the kibana executors, skipping .buildkite/scripts/lifecycle/pre_command.sh"
-  export SONAR_LOGIN=$(vault read -field=token secret/ci/elastic-kibana/sonarqube)
-else
-  source .buildkite/scripts/lifecycle/pre_command.sh
-fi
+source .buildkite/scripts/lifecycle/pre_command.sh
+export SONAR_LOGIN=$(vault read -field=token secret/ci/elastic-kibana/sonarqube)
+

.buildkite/pipeline-utils/buildkite/client.ts

@@ -36,7 +36,13 @@ export interface BuildkiteCommandStep {
   label: string;
   parallelism?: number;
   agents: {
-    queue: string;
+    queue?: string;
+    provider?: string;
+    image?: string;
+    imageProject?: string;
+    machineType?: string;
+    minCpuPlatform?: string;
+    preemptible?: boolean;
   };
   timeout_in_minutes?: number;
   key?: string;

.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts

@@ -419,7 +419,11 @@ export async function pickTestGroupRunOrder() {
             timeout_in_minutes: 120,
             key: 'jest',
             agents: {
-              queue: 'n2-4-spot',
+              provider: 'gcp',
+              image: 'family/kibana-ubuntu-2004',
+              imageProject: 'elastic-images-qa',
+              machineType: 'n2-standard-4',
+              preemptible: true,
             },
             retry: {
               automatic: [
@@ -439,7 +443,11 @@ export async function pickTestGroupRunOrder() {
             timeout_in_minutes: 120,
             key: 'jest-integration',
             agents: {
-              queue: 'n2-4-spot',
+              provider: 'gcp',
+              image: 'family/kibana-ubuntu-2004',
+              imageProject: 'elastic-images-qa',
+              machineType: 'n2-standard-4',
+              preemptible: true,
             },
             retry: {
               automatic: [
@@ -470,12 +478,17 @@ export async function pickTestGroupRunOrder() {
                   : -1
               )
               .map(
-                ({ title, key, queue = defaultQueue }): BuildkiteStep => ({
+                // ({ title, key, queue = defaultQueue }): BuildkiteStep => ({
+                ({ title, key }): BuildkiteStep => ({
                   label: title,
                   command: getRequiredEnv('FTR_CONFIGS_SCRIPT'),
                   timeout_in_minutes: 90,
                   agents: {
-                    queue,
+                    provider: 'gcp',
+                    image: 'family/kibana-ubuntu-2004',
+                    imageProject: 'elastic-images-qa',
+                    machineType: 'n2-standard-4',
+                    preemptible: true,
                   },
                   env: {
                     FTR_CONFIG_GROUP_KEY: key,

.buildkite/pipelines/artifacts.yml

@@ -2,7 +2,12 @@ steps:
   - command: .buildkite/scripts/steps/artifacts/build.sh
     label: Build Kibana Artifacts
     agents:
-      queue: c2-16
+      provider: gcp
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      machineType: c2-standard-16
+      buildDirectory: /dev/shm/builds
+      assignExternalIP: false
     timeout_in_minutes: 60
     retry:
       automatic:
@@ -14,7 +19,13 @@ steps:
   - command: TEST_PACKAGE=deb .buildkite/scripts/steps/package_testing/test.sh
     label: Artifact Testing
     agents:
-      queue: n2-4-virt
+      provider: gcp
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      enableNestedVirtualization: true
+      localSsds: 1
+      machineType: n2-standard-4
+      assignExternalIP: false
     timeout_in_minutes: 30
     retry:
       automatic:
@@ -24,7 +35,13 @@ steps:
   - command: TEST_PACKAGE=rpm .buildkite/scripts/steps/package_testing/test.sh
     label: Artifact Testing
     agents:
-      queue: n2-4-virt
+      provider: gcp
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      enableNestedVirtualization: true
+      localSsds: 1
+      machineType: n2-standard-4
+      assignExternalIP: false
     timeout_in_minutes: 30
     retry:
       automatic:
@@ -34,7 +51,13 @@ steps:
   - command: TEST_PACKAGE=docker .buildkite/scripts/steps/package_testing/test.sh
     label: Artifact Testing
     agents:
-      queue: n2-4-virt
+      provider: gcp
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      enableNestedVirtualization: true
+      localSsds: 1
+      machineType: n2-standard-4
+      assignExternalIP: false
     timeout_in_minutes: 30
     retry:
       automatic:
@@ -44,7 +67,27 @@ steps:
   - command: KIBANA_DOCKER_CONTEXT=default .buildkite/scripts/steps/artifacts/docker_context.sh
     label: 'Docker Context Verification'
     agents:
-      queue: n2-2
+      provider: gcp
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      localSsds: 1
+      machineType: n2-standard-2
+      assignExternalIP: false
+    timeout_in_minutes: 30
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: KIBANA_DOCKER_CONTEXT=ubi8 .buildkite/scripts/steps/artifacts/docker_context.sh
+    label: 'Docker Context Verification'
+    agents:
+      provider: gcp
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      localSsds: 1
+      machineType: n2-standard-2
+      assignExternalIP: false
     timeout_in_minutes: 30
     retry:
       automatic:
@@ -54,7 +97,12 @@ steps:
   - command: KIBANA_DOCKER_CONTEXT=ubi .buildkite/scripts/steps/artifacts/docker_context.sh
     label: 'Docker Context Verification'
     agents:
-      queue: n2-2
+      provider: gcp
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      localSsds: 1
+      machineType: n2-standard-2
+      assignExternalIP: false
     timeout_in_minutes: 30
     retry:
       automatic:
@@ -64,7 +112,12 @@ steps:
   - command: KIBANA_DOCKER_CONTEXT=ironbank .buildkite/scripts/steps/artifacts/docker_context.sh
     label: 'Docker Context Verification'
     agents:
-      queue: n2-2
+      provider: gcp
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      localSsds: 1
+      machineType: n2-standard-2
+      assignExternalIP: false
     timeout_in_minutes: 30
     retry:
       automatic:
@@ -77,7 +130,12 @@ steps:
       - exit_status: 255
       - exit_status: -1
     agents:
-      queue: n2-2
+      provider: gcp
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      localSsds: 1
+      machineType: n2-standard-2
+      assignExternalIP: false
     timeout_in_minutes: 60
     if: "build.env('RELEASE_BUILD') == null || build.env('RELEASE_BUILD') == '' || build.env('RELEASE_BUILD') == 'false'"
     retry:
@@ -99,5 +157,10 @@ steps:
   - command: .buildkite/scripts/steps/artifacts/publish.sh
     label: 'Publish Kibana Artifacts'
     agents:
-      queue: n2-2
+      provider: gcp
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      localSsds: 1
+      machineType: n2-standard-2
+      assignExternalIP: false
     timeout_in_minutes: 30

.buildkite/pipelines/artifacts_container_image.yml

@@ -2,7 +2,12 @@ steps:
   - command: .buildkite/scripts/steps/artifacts/docker_image.sh
     label: Build serverless container images
     agents:
-      queue: n2-16-spot
+      provider: gcp
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      machineType: n2-standard-16
+      preemptible: true
+      buildDirectory: /dev/shm/builds
     timeout_in_minutes: 60
     retry:
       automatic:

.buildkite/pipelines/artifacts_trigger.yml

@@ -2,5 +2,9 @@ steps:
   - command: .buildkite/scripts/steps/artifacts/trigger.sh
     label: Trigger artifacts build
     agents:
-      queue: kibana-default
+      provider: gcp
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      machineType: n2-standard-2
+      assignExternalIP: false
     timeout_in_minutes: 10

.buildkite/pipelines/bazel_cache.yml

@@ -1,7 +1,11 @@
 steps:
   - label: ':pipeline: Create pipeline with priority'
     agents:
-      queue: kibana-default
+      provider: gcp
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      machineType: n2-standard-2
+      assignExternalIP: false
     concurrency_group: bazel_macos
     concurrency: 1
     concurrency_method: eager

.buildkite/pipelines/build_api_docs.yml

@@ -4,7 +4,11 @@ steps:
   - command: .buildkite/scripts/steps/build_api_docs.sh
     label: 'Build API Docs'
     agents:
-      queue: n2-4-spot
+      provider: gcp
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      machineType: n2-standard-4
+      preemptible: true
     key: build_api_docs
     timeout_in_minutes: 60
     retry:

.buildkite/pipelines/code_coverage/daily.yml

@@ -3,14 +3,22 @@ steps:
     label: Pre-Build
     timeout_in_minutes: 10
     agents:
-      queue: kibana-default
+      provider: gcp
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      machineType: n2-standard-2
+      assignExternalIP: false
 
   - wait
 
   - command: .buildkite/scripts/steps/test/pick_test_group_run_order.sh
     label: 'Pick Test Group Run Order'
     agents:
-      queue: kibana-default
+      provider: gcp
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      machineType: n2-standard-2
+      assignExternalIP: false
     timeout_in_minutes: 10
     env:
       FTR_CONFIGS_DEPS: ''
@@ -21,7 +29,11 @@ steps:
   - command: .buildkite/scripts/steps/code_coverage/ingest.sh
     label: 'Merge and Ingest'
     agents:
-      queue: n2-4-spot
+      provider: gcp
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      machineType: n2-standard-4
+      preemptible: true
     depends_on:
       - jest
       - jest-integration

.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml

@@ -10,7 +10,11 @@
 # BUILDKITE_COMMIT: the commit hash of the kibana branch to test
 
 agents:
-  queue: kibana-default
+  provider: gcp
+  image: family/kibana-ubuntu-2004
+  imageProject: elastic-images-qa
+  machineType: n2-standard-2
+  assignExternalIP: false
 
 steps:
   - label: "Annotate runtime parameters"
@@ -26,7 +30,11 @@ steps:
         key: pre-build
         timeout_in_minutes: 10
         agents:
-          queue: kibana-default
+          provider: gcp
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-qa
+          machineType: n2-standard-2
+          assignExternalIP: false
 
       - label: "Build Kibana Distribution and Plugins"
         command: .buildkite/scripts/steps/build_kibana.sh
@@ -43,7 +51,11 @@ steps:
       - label: "Pick Test Group Run Order"
         command: .buildkite/scripts/steps/test/pick_test_group_run_order.sh
         agents:
-          queue: kibana-default
+          provider: gcp
+          image: family/kibana-ubuntu-2004
+          imageProject: elastic-images-qa
+          machineType: n2-standard-2
+          assignExternalIP: false
         depends_on: build
         timeout_in_minutes: 10
         env:
@@ -197,4 +209,8 @@ steps:
     command: .buildkite/scripts/lifecycle/post_build.sh
     timeout_in_minutes: 10
     agents:
-      queue: kibana-default
+      provider: gcp
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      machineType: n2-standard-2
+      assignExternalIP: false

.buildkite/pipelines/es_snapshots/build.yml

@@ -3,4 +3,9 @@ steps:
     label: Build ES Snapshot
     timeout_in_minutes: 30
     agents:
-      queue: c2-8
+      provider: gcp
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      localSsds: 1
+      machineType: c2-standard-8
+      assignExternalIP: false

.buildkite/pipelines/es_snapshots/promote.yml

@@ -11,4 +11,8 @@ steps:
   - label: Promote Snapshot
     command: .buildkite/scripts/steps/es_snapshots/promote.sh
     agents:
-      queue: kibana-default
+      provider: gcp
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      machineType: n2-standard-2
+      assignExternalIP: false