Skip to content

Commit

Permalink
Merge branch 'main' into use-authorisation-if-present
Browse files Browse the repository at this point in the history
  • Loading branch information
kibanamachine authored Aug 18, 2022
2 parents cb7104b + 209f933 commit e138d5a
Show file tree
Hide file tree
Showing 2,505 changed files with 71,249 additions and 39,810 deletions.
79 changes: 0 additions & 79 deletions .buildkite/agents.json

This file was deleted.

2 changes: 1 addition & 1 deletion .buildkite/pipeline-utils/test-failures/annotate.ts
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,7 @@ export const annotateTestFailures = async () => {
}

exec(
`buildkite-agent artifact download --include-retried-jobs "target/test_failures/*.json" "${failureDir}"`
`.buildkite/scripts/common/download_artifact.sh --include-retried-jobs "target/test_failures/*.json" "${failureDir}"`
);

const failures: TestFailure[] = recursiveReadDir(failureDir)
Expand Down
6 changes: 6 additions & 0 deletions .buildkite/scripts/common/download_artifact.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
#!/usr/bin/env bash

set -euo pipefail

source "$(dirname "$0")/util.sh"
download_artifact "$@"
20 changes: 16 additions & 4 deletions .buildkite/scripts/common/env.sh
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,22 @@ export KIBANA_BASE_BRANCH="$KIBANA_PKG_BRANCH"
KIBANA_PKG_VERSION="$(jq -r .version "$KIBANA_DIR/package.json")"
export KIBANA_PKG_VERSION

export GECKODRIVER_CDNURL="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache"
export CHROMEDRIVER_CDNURL="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache"
export RE2_DOWNLOAD_MIRROR="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache"
export CYPRESS_DOWNLOAD_MIRROR="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache/cypress"
BUILDKITE_AGENT_GCP_REGION=""
if [[ "$(curl -is metadata.google.internal || true)" ]]; then
# projects/1003139005402/zones/us-central1-a -> us-central1-a -> us-central1
BUILDKITE_AGENT_GCP_REGION=$(curl -sH Metadata-Flavor:Google http://metadata.google.internal/computeMetadata/v1/instance/zone | rev | cut -d'/' -f1 | cut -c3- | rev)
fi
export BUILDKITE_AGENT_GCP_REGION

CI_PROXY_CACHE_SUFFIX=""
if [[ "$BUILDKITE_AGENT_GCP_REGION" ]]; then
CI_PROXY_CACHE_SUFFIX="/region/$BUILDKITE_AGENT_GCP_REGION"
fi

export GECKODRIVER_CDNURL="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache$CI_PROXY_CACHE_SUFFIX"
export CHROMEDRIVER_CDNURL="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache$CI_PROXY_CACHE_SUFFIX"
export RE2_DOWNLOAD_MIRROR="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache$CI_PROXY_CACHE_SUFFIX"
export CYPRESS_DOWNLOAD_MIRROR="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache$CI_PROXY_CACHE_SUFFIX/cypress"

export NODE_OPTIONS="--max-old-space-size=4096"

Expand Down
7 changes: 1 addition & 6 deletions .buildkite/scripts/common/setup_bazel.sh
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,7 @@ BAZEL_CACHE_MODE=${BAZEL_CACHE_MODE:-gcs}
if [[ "$BAZEL_CACHE_MODE" == "gcs" ]]; then
echo "[bazel] enabling caching with GCS buckets"

BAZEL_REGION="us-central1"
if [[ "$(curl -is metadata.google.internal || true)" ]]; then
# projects/1003139005402/zones/us-central1-a -> us-central1-a -> us-central1
BAZEL_REGION=$(curl -sH Metadata-Flavor:Google http://metadata.google.internal/computeMetadata/v1/instance/zone | rev | cut -d'/' -f1 | cut -c3- | rev)
fi

BAZEL_REGION="${BUILDKITE_AGENT_GCP_REGION:-us-central1}"
BAZEL_BUCKET="kibana-ci-bazel_$BAZEL_REGION"

echo "[bazel] using GCS bucket: $BAZEL_BUCKET"
Expand Down
6 changes: 6 additions & 0 deletions .buildkite/scripts/common/util.sh
Original file line number Diff line number Diff line change
Expand Up @@ -168,3 +168,9 @@ npm_install_global() {
npm install -g "$toInstall" && touch "$successFlag"
fi
}

# Download an artifact using the buildkite-agent, takes the same arguments as https://buildkite.com/docs/agent/v3/cli-artifact#downloading-artifacts-usage
# times-out after 60 seconds and retries up to 3 times
download_artifact() {
retry 3 1 timeout 3m buildkite-agent artifact download "$@"
}
6 changes: 4 additions & 2 deletions .buildkite/scripts/download_build_artifacts.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,15 @@

set -euo pipefail

source "$(dirname "$0")/common/util.sh"

if [[ ! -d "$KIBANA_BUILD_LOCATION/bin" ]]; then
echo '--- Downloading Distribution and Plugin artifacts'

cd "$WORKSPACE"

buildkite-agent artifact download kibana-default.tar.gz . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
buildkite-agent artifact download kibana-default-plugins.tar.gz . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
download_artifact kibana-default.tar.gz . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
download_artifact kibana-default-plugins.tar.gz . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"

mkdir -p "$KIBANA_BUILD_LOCATION"
tar -xzf kibana-default.tar.gz -C "$KIBANA_BUILD_LOCATION" --strip=1
Expand Down
3 changes: 2 additions & 1 deletion .buildkite/scripts/steps/artifacts/cloud.sh
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,13 @@ set -euo pipefail

.buildkite/scripts/bootstrap.sh

source "$(dirname "$0")/../../common/util.sh"
source .buildkite/scripts/steps/artifacts/env.sh

echo "--- Build and publish Cloud image"
mkdir -p target

buildkite-agent artifact download "kibana-$FULL_VERSION-linux-x86_64.tar.gz" ./target --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
download_artifact "kibana-$FULL_VERSION-linux-x86_64.tar.gz" ./target --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"

node scripts/build \
--skip-initialize \
Expand Down
3 changes: 2 additions & 1 deletion .buildkite/scripts/steps/artifacts/docker_context.sh
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ set -euo pipefail

.buildkite/scripts/bootstrap.sh

source "$(dirname "$0")/../../common/util.sh"
source .buildkite/scripts/steps/artifacts/env.sh

KIBANA_DOCKER_CONTEXT="${KIBANA_DOCKER_CONTEXT:="default"}"
Expand All @@ -28,7 +29,7 @@ fi
tar -xf "target/$DOCKER_CONTEXT_FILE" -C "$DOCKER_BUILD_FOLDER"
cd $DOCKER_BUILD_FOLDER

buildkite-agent artifact download "kibana-$FULL_VERSION-linux-x86_64.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
download_artifact "kibana-$FULL_VERSION-linux-x86_64.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"

echo "--- Build context"
docker build .
6 changes: 3 additions & 3 deletions .buildkite/scripts/steps/artifacts/publish.sh
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@ source .buildkite/scripts/steps/artifacts/env.sh

echo "--- Download and verify artifacts"
function download {
buildkite-agent artifact download "$1" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
buildkite-agent artifact download "$1.sha512.txt" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
download_artifact "$1" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
download_artifact "$1.sha512.txt" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
sha512sum -c "$1.sha512.txt"
rm "$1.sha512.txt"
}
Expand Down Expand Up @@ -40,7 +40,7 @@ download "kibana-$FULL_VERSION-windows-x86_64.zip"

download "dependencies-$FULL_VERSION.csv"

cd -
cd -

echo "--- Set artifact permissions"
chmod -R a+r target/*
Expand Down
2 changes: 1 addition & 1 deletion .buildkite/scripts/steps/cloud/build_and_deploy.sh
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ ECCTL_LOGS=$(mktemp --suffix ".json")
echo "--- Download Kibana Distribution"

mkdir -p ./target
buildkite-agent artifact download "kibana-$VERSION-linux-x86_64.tar.gz" ./target --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
download_artifact "kibana-$VERSION-linux-x86_64.tar.gz" ./target --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"

echo "--- Build Cloud Distribution"
ELASTICSEARCH_MANIFEST_URL="https://storage.googleapis.com/kibana-ci-es-snapshots-daily/$(jq -r '.version' package.json)/manifest-latest-verified.json"
Expand Down
2 changes: 1 addition & 1 deletion .buildkite/scripts/steps/code_coverage/ftr_configs.sh
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ configs="${FTR_CONFIG:-}"

if [[ "$configs" == "" ]]; then
echo "--- Downloading ftr test run order"
buildkite-agent artifact download ftr_run_order.json .
download_artifact ftr_run_order.json .
configs=$(jq -r '.[env.FTR_CONFIG_GROUP_KEY].names[]' ftr_run_order.json)
fi

Expand Down
4 changes: 2 additions & 2 deletions .buildkite/scripts/steps/code_coverage/ingest.sh
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ revolveBuildHashes() {
}

collectRan() {
buildkite-agent artifact download target/ran_files/* .
download_artifact target/ran_files/* .

while read -r x; do
ran=("${ran[@]}" "$(cat "$x")")
Expand All @@ -51,7 +51,7 @@ fetchArtifacts() {

local xs=("$@")
for x in "${xs[@]}"; do
buildkite-agent artifact download "target/kibana-coverage/${x}/*" .
download_artifact "target/kibana-coverage/${x}/*" .
done
}

Expand Down
3 changes: 2 additions & 1 deletion .buildkite/scripts/steps/code_coverage/jest_parallel.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

set -euo pipefail

source "$(dirname "$0")/../../common/util.sh"
source .buildkite/scripts/steps/code_coverage/util.sh

export JOB=$BUILDKITE_PARALLEL_JOB
Expand All @@ -18,7 +19,7 @@ fi

export TEST_TYPE
echo "--- downloading jest test run order"
buildkite-agent artifact download jest_run_order.json .
download_artifact jest_run_order.json .
configs=$(jq -r 'getpath([env.TEST_TYPE]) | .groups[env.JOB | tonumber].names | .[]' jest_run_order.json)

echo "--- KIBANA_DIR: $KIBANA_DIR"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,8 @@ mkdir "${BUILD_ID}"
tar -czf "${BUILD_ID}/scalability_traces.tar.gz" -C target scalability_traces
buildkite-agent artifact upload "${BUILD_ID}/scalability_traces.tar.gz"
# Upload Kibana build, plugins, commit sha and traces to the bucket
buildkite-agent artifact download kibana-default.tar.gz ./"${BUILD_ID}"
buildkite-agent artifact download kibana-default-plugins.tar.gz ./"${BUILD_ID}"
download_artifact kibana-default.tar.gz ./"${BUILD_ID}"
download_artifact kibana-default-plugins.tar.gz ./"${BUILD_ID}"
echo "${BUILDKITE_COMMIT}" > "${BUILD_ID}/KIBANA_COMMIT_HASH"
gsutil -m cp -r "${BUILD_ID}" "${GCS_BUCKET}"
echo "--- Update reference to the latest CI build"
Expand Down
7 changes: 4 additions & 3 deletions .buildkite/scripts/steps/package_testing/test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,20 +2,21 @@

set -euo pipefail

source "$(dirname "$0")/../../common/util.sh"
.buildkite/scripts/bootstrap.sh

echo "--- Package Testing for $TEST_PACKAGE"

mkdir -p target
cd target
if [[ "$TEST_PACKAGE" == "deb" ]]; then
buildkite-agent artifact download 'kibana-*-amd64.deb' . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
download_artifact 'kibana-*-amd64.deb' . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
KIBANA_IP_ADDRESS="192.168.56.5"
elif [[ "$TEST_PACKAGE" == "rpm" ]]; then
buildkite-agent artifact download 'kibana-*-x86_64.rpm' . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
download_artifact 'kibana-*-x86_64.rpm' . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
KIBANA_IP_ADDRESS="192.168.56.6"
elif [[ "$TEST_PACKAGE" == "docker" ]]; then
buildkite-agent artifact download "kibana-$KIBANA_PKG_VERSION*-docker-image.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
download_artifact "kibana-$KIBANA_PKG_VERSION*-docker-image.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
KIBANA_IP_ADDRESS="192.168.56.7"
fi
cd ..
Expand Down
1 change: 1 addition & 0 deletions .buildkite/scripts/steps/storybooks/build_and_upload.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ const STORYBOOKS = [
'ci_composite',
'cloud',
'coloring',
'chart_icons',
'controls',
'custom_integrations',
'dashboard_enhanced',
Expand Down
2 changes: 1 addition & 1 deletion .buildkite/scripts/steps/test/ftr_configs.sh
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ fi

if [ "$configs" == "" ] && [ "$FTR_CONFIG_GROUP_KEY" != "" ]; then
echo "--- downloading ftr test run order"
buildkite-agent artifact download ftr_run_order.json .
download_artifact ftr_run_order.json .
configs=$(jq -r '.[env.FTR_CONFIG_GROUP_KEY].names[]' ftr_run_order.json)
fi

Expand Down
3 changes: 2 additions & 1 deletion .buildkite/scripts/steps/test/jest_parallel.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

set -euo pipefail

source "$(dirname "$0")/../../common/util.sh"
export JOB=${BUILDKITE_PARALLEL_JOB:-0}

# a jest failure will result in the script returning an exit code of 10
Expand All @@ -20,7 +21,7 @@ fi

export TEST_TYPE
echo "--- downloading jest test run order"
buildkite-agent artifact download jest_run_order.json .
download_artifact jest_run_order.json .
configs=$(jq -r 'getpath([env.TEST_TYPE]) | .groups[env.JOB | tonumber].names | .[]' jest_run_order.json)

while read -r config; do
Expand Down
Loading

0 comments on commit e138d5a

Please sign in to comment.