Remove hardcoded module definitions in CI

For reasons described in [^1], we hard coded `aws` or `kubernetes` for the env var `MODULE`, when no file changes were detected under `modules/` per project. This commit reverts this and brings back the default behavior which is not to set the MODULE env var when nothing has changed. Turns out that module tests also require the right GOX_FLAGS. This PR sets the GOX_FLAGS mirroring the Jenkins setup[^2]. [^1]: elastic/ingest-dev#2993 [^2]: https://github.com/elastic/beats/blob/0371883dffd6fba0c01e164aeee8b412f1681bff/Jenkinsfile#L642
elastic · May 10, 2024 · f8a5d80 · f8a5d80
1 parent 760208b
commit f8a5d80
Show file tree

Hide file tree

Showing 8 changed files with 40 additions and 24 deletions.
diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
@@ -4,6 +4,37 @@ set -euo pipefail
 
 source .buildkite/env-scripts/util.sh
 
+isUnix() {
+  # return 0 if it's Linux or macOS and 1 if anything else (e.g. Windows)
+  local OS=$(uname -o | tr '[:upper:]' '[:lower:]')
+  if [[ $OS == *"linux"* || $OS == *"darwin"* ]]; then
+    return 0
+  else
+    return 1
+  fi
+}
+
+isArm() {
+  # return 0 if it's arm architecture (32 or 64bit), 1 otherwise
+  local MACHINE=$(uname -m | tr '[:upper:]' '[:lower:]')
+  if [[ $MACHINE == *"arm"* || $MACHINE == *"aarch"* ]]; then
+    return 0
+  else
+    return 1
+  fi
+}
+
+if isUnix && isArm; then
+  export GOX_FLAGS="-arch arm"
+elif isUnix; then
+  export GOX_FLAGS="-arch amd64"
+else
+  # windows etc.
+  export GOX_FLAGS="-arch amd64"
+fi
+
+echo "~~~ Setting env var GOX_FLAGS to [$GOX_FLAGS]"
+
 # Secrets must be redacted
 # https://buildkite.com/docs/pipelines/managing-log-output#redacted-environment-variables
 AWS_SERVICE_ACCOUNT_SECRET_PATH="kv/ci-shared/platform-ingest/aws_account_auth"

diff --git a/.buildkite/metricbeat/pipeline.yml b/.buildkite/metricbeat/pipeline.yml
@@ -53,7 +53,6 @@ steps:
           # defines the MODULE env var based on what's changed in a PR
           source .buildkite/scripts/changesets.sh
           defineModuleFromTheChangeSet metricbeat
-          echo "~~~ Will run tests with env var MODULE=$$MODULE"
 
           # TODO move this section to base image / pre-command hook
           echo "~~~ Installing kind"
@@ -86,7 +85,7 @@ steps:
           # defines the MODULE env var based on what's changed in a PR
           source .buildkite/scripts/changesets.sh
           defineModuleFromTheChangeSet metricbeat
-          echo "~~~ Running tests with env var MODULE=$$MODULE"
+          echo "~~~ Running tests"
 
           # TODO move this section to base image / pre-command hook
           echo "~~~ Installing kind"

diff --git a/.buildkite/scripts/changesets.sh b/.buildkite/scripts/changesets.sh
@@ -68,13 +68,10 @@ defineModuleFromTheChangeSet() {
     fi
   done
 
-  if [[ -z "$changed_modules" ]]; then # TODO: remove this conditional when issue https://github.com/elastic/ingest-dev/issues/2993 gets resolved
-    if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then
-      export MODULE="aws"
-    else
-      export MODULE="kubernetes"
-    fi
-  else
-    export MODULE="${changed_modules}"  # TODO: once https://github.com/elastic/ingest-dev/issues/2993 gets resolved, this should be the only thing we export
+  # export MODULE="" leads to an infinite loop https://github.com/elastic/ingest-dev/issues/2993
+  if [[ ! -z $changed_modules ]]; then 
+    export MODULE="${changed_modules}"
+    echo "~~~ Set env var MODULE to [$MODULE]"
+    echo "~~~ Resuming commands"
   fi
 }
diff --git a/.buildkite/x-pack/pipeline.xpack.auditbeat.yml b/.buildkite/x-pack/pipeline.xpack.auditbeat.yml
@@ -33,7 +33,6 @@ steps:
           # defines the MODULE env var based on what's changed in a PR
           source .buildkite/scripts/changesets.sh
           defineModuleFromTheChangeSet x-pack/auditbeat
-          echo "~~~ Will run tests with env var MODULE=$$MODULE"
           cd x-pack/auditbeat
           mage update build test
         retry:

diff --git a/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml b/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml
@@ -52,11 +52,8 @@ steps:
           # defines the MODULE env var based on what's changed in a PR
           source .buildkite/scripts/changesets.sh
           defineModuleFromTheChangeSet x-pack/dockerlogbeat
-          echo "~~~ Running tests with env var MODULE=$$MODULE"
           cd x-pack/dockerlogbeat
           mage goIntegTest
-        env:
-          MODULE: $MODULE
         retry:
           automatic:
            - limit: 3

diff --git a/.buildkite/x-pack/pipeline.xpack.filebeat.yml b/.buildkite/x-pack/pipeline.xpack.filebeat.yml
@@ -51,7 +51,6 @@ steps:
           # defines the MODULE env var based on what's changed in a PR
           source .buildkite/scripts/changesets.sh
           defineModuleFromTheChangeSet x-pack/filebeat
-          echo "~~~ Will run tests with env var MODULE=$$MODULE"
           cd x-pack/filebeat && mage goIntegTest
         retry:
           automatic:
@@ -74,7 +73,6 @@ steps:
           # defines the MODULE env var based on what's changed in a PR
           source .buildkite/scripts/changesets.sh
           defineModuleFromTheChangeSet x-pack/filebeat
-          echo "~~~ Running tests with env var MODULE=$$MODULE"
           cd x-pack/filebeat && mage pythonIntegTest
         retry:
           automatic:
@@ -268,7 +266,7 @@ steps:
           # defines the MODULE env var based on what's changed in a PR
           source .buildkite/scripts/changesets.sh
           defineModuleFromTheChangeSet x-pack/filebeat
-          echo "~~~ Running tests with env var MODULE=$$MODULE"
+          echo "~~~ Running tests"
 
           .buildkite/scripts/cloud_tests.sh
         env:
@@ -297,7 +295,6 @@ steps:
           # defines the MODULE env var based on what's changed in a PR
           source .buildkite/scripts/changesets.sh
           defineModuleFromTheChangeSet x-pack/filebeat
-          echo "~~~ Running tests with env var MODULE=$$MODULE"
           .buildkite/scripts/cloud_tests.sh
         env:
           ASDF_TERRAFORM_VERSION: 1.0.2

diff --git a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml
@@ -51,7 +51,6 @@ steps:
           # defines the MODULE env var based on what's changed in a PR
           source .buildkite/scripts/changesets.sh
           defineModuleFromTheChangeSet x-pack/metricbeat
-          echo "~~~ Will run tests with env var MODULE=$$MODULE"
           cd x-pack/metricbeat && mage goIntegTest
         retry:
           automatic:
@@ -74,7 +73,7 @@ steps:
           # defines the MODULE env var based on what's changed in a PR
           source .buildkite/scripts/changesets.sh
           defineModuleFromTheChangeSet x-pack/metricbeat
-          echo "~~~ Running tests with env var MODULE=$$MODULE"
+          echo "~~~ Running tests"
           cd x-pack/metricbeat && mage pythonIntegTest
         retry:
           automatic:
@@ -253,7 +252,7 @@ steps:
           # defines the MODULE env var based on what's changed in a PR
           source .buildkite/scripts/changesets.sh
           defineModuleFromTheChangeSet x-pack/metricbeat
-          echo "~~~ Running tests with env var MODULE=$$MODULE"
+          echo "~~~ Running tests"
 
           .buildkite/scripts/cloud_tests.sh
         env:
@@ -284,7 +283,6 @@ steps:
           # defines the MODULE env var based on what's changed in a PR
           source .buildkite/scripts/changesets.sh
           defineModuleFromTheChangeSet x-pack/metricbeat
-          echo "~~~ Running tests with env var MODULE=$$MODULE"
           .buildkite/scripts/cloud_tests.sh
         env:
           ASDF_TERRAFORM_VERSION: 1.0.2

diff --git a/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml b/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml
@@ -27,8 +27,6 @@ steps:
           Write-Output "~~~ Will run tests with env var MODULE=$$Env:MODULE"
           Set-Location -Path x-pack/winlogbeat
           mage build unitTest
-        env:
-          MODULE: $MODULE
         retry:
           automatic:
            - limit: 3