[ci] Backport Jenkinsfile and supporting changes (#17225)

* Backport Jenkinsfile and supporting scripts

Files are from b82b9db

* [ci] Make tests pass in Jenkinsfile pipeline (#17071)

* Address failing tests and warnings

* Ensure mage unitTest works consistently

Windows will invoke `mage unitTest` that will run the Go and Python tests without requiring the integration tests environment.

Use an alias for goTestUnit in metricbeat

* Only include test output when `mage -v` is used

The full test output is written to a file and we archive that. Additionally if a test fails that output is still logged out stderr.

This helps speed things up because Jenkins doesn't need to deal with streaming megabytes of log output to the console.
It also makes it easier to spot errors in the log.

* Jenkinsfile updates

Changes:

- Disable macOS by default. These workers don't have python3 so they cannot a portion of the tests.
- Add python38 to the PATH on Windows.
- Various fixes to the environment variables on Windows.
- Always add the GVM GOROOT/bin to the PATH on POSIX. Same as was being done on Windows.
- Use bat instead of sh on Windows to run `mage dumpVariables`.

* Add MAGEFILE_VERBOSE to travis-ci env

.ci/scripts/install-docker-compose.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+set -exuo pipefail
+
+MSG="parameter missing."
+DOCKER_COMPOSE_VERSION=${DOCKER_COMPOSE_VERSION:?$MSG}
+HOME=${HOME:?$MSG}
+DC_CMD="${HOME}/bin/docker-compose"
+
+mkdir -p "${HOME}/bin"
+
+curl -sSLo "${DC_CMD}" "https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-$(uname -s)-$(uname -m)"
+chmod +x "${DC_CMD}"

.ci/scripts/install-go.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+set -exuo pipefail
+
+MSG="parameter missing."
+GO_VERSION=${GO_VERSION:?$MSG}
+PROPERTIES_FILE=${PROPERTIES_FILE:-"go_env.properties"}
+HOME=${HOME:?$MSG}
+ARCH=$(uname -s| tr '[:upper:]' '[:lower:]')
+GVM_CMD="${HOME}/bin/gvm"
+
+mkdir -p "${HOME}/bin"
+
+curl -sSLo "${GVM_CMD}" "https://github.com/andrewkroh/gvm/releases/download/v0.2.2/gvm-${ARCH}-amd64"
+chmod +x "${GVM_CMD}"
+
+gvm ${GO_VERSION}|cut -d ' ' -f 2|tr -d '\"' > ${PROPERTIES_FILE}
+
+eval $(gvm ${GO_VERSION})

.ci/scripts/install-tools.bat

@@ -0,0 +1,27 @@
+set GOPATH=%WORKSPACE%
+set MAGEFILE_CACHE=%WORKSPACE%\.magefile
+set PATH=%WORKSPACE%\bin;C:\ProgramData\chocolatey\bin;%PATH%
+
+where /q curl
+IF ERRORLEVEL 1 (
+ choco install curl -y --no-progress --skipdownloadcache
+)
+mkdir %WORKSPACE%\bin
+where /q gvm
+IF ERRORLEVEL 1 (
+ curl -sL -o %WORKSPACE%\bin\gvm.exe https://github.com/andrewkroh/gvm/releases/download/v0.2.2/gvm-windows-amd64.exe
+)
+FOR /f "tokens=*" %%i IN ('"gvm.exe" use %GO_VERSION% --format=batch') DO %%i
+
+go env
+go install -mod=vendor github.com/magefile/mage
+mage -version
+where mage
+
+if not exist C:\Python38\python.exe (
+    REM Install python 3.8.
+    choco install python -y -r --no-progress --version 3.8.2
+)
+python --version
+where python
+

.ci/scripts/kind-setup.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+set -exuo pipefail
+
+MSG="parameter missing."
+K8S_VERSION=${K8S_VERSION:?$MSG}
+HOME=${HOME:?$MSG}
+KBC_CMD="${HOME}/bin/kubectl"
+
+mkdir -p "${HOME}/bin"
+
+curl -sSLo "${KBC_CMD}" "https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/linux/amd64/kubectl"
+chmod +x "${KBC_CMD}"
+
+GO111MODULE="on" go get sigs.k8s.io/[email protected]
+kind create cluster --image kindest/node:${K8S_VERSION}
+
+export KUBECONFIG="$(kind get kubeconfig-path)"
+kubectl cluster-info

.editorconfig

@@ -29,6 +29,10 @@ indent_style = tab
 [*.mk]
 indent_style = tab
 
+[Jenkinsfile]
+indent_size = 2
+indent_style = space
+
 [Vagrantfile]
 indent_size = 2
 indent_style = space

.travis.yml

@@ -17,6 +17,9 @@ env:
     # Newer versions of minikube fail on travis, see: https://github.com/kubernetes/minikube/issues/2704
     - TRAVIS_MINIKUBE_VERSION=v0.25.2
     - MACOSX_DEPLOYMENT_TARGET=10.15
+    # Enable verbose output since Travis users cannot access test output that is written to files.
+    # The constant output from tests also prevents time-outs.
+    - MAGEFILE_VERBOSE=true
 
 # Only run CI jobs on specific branches.
 # To keep the number of Travis CI jobs in check, we decided to configure Travis to only kick off CI jobs for the master branch.