forked from elastic/kibana
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'master' of github.com:elastic/kibana into reporting/add…
…-link-to-kibana-app * 'master' of github.com:elastic/kibana: (120 commits) [TSVB] Support custom field format (elastic#101245) [VisEditors] Add code ownership to the functional tests (elastic#111680) [Lens] Make Lens saved object share-capable (elastic#111403) [Graph] Make Graph saved object share-capable (elastic#111404) [Stack Monitoring] Add breadcrumb support (elastic#111850) Update Jira Cloud to use OAuth2.0 (elastic#111493) Show warning message when attempting to create an APM alert in stack management (elastic#111781) Skip suite blocking ES snapshot promotion (elastic#111907) Respect `auth_provider_hint` if session is not authenticated. (elastic#111521) Added in 'Responses' field in alert telemetry & updated test (elastic#111892) [Usage collection] refactor cloud detector collector (elastic#110439) Make classnames a shared dep (elastic#111636) Fix link to e2e tests in APM testing.md (elastic#111869) [Security Solution] Add host.os.name.caseless mapping and runtime field (elastic#111455) [APM] Removes the beta label from APM tutorial (elastic#111499) (elastic#111828) [RAC] [Observability] Expand Observability alerts page functional tests (elastic#111297) Fix extra white space on the alert table whe page size is 50 or 100 (elastic#111568) [Metrics UI] Add Inventory Timeline open/close state to context and URL state (elastic#111034) [Graph] Switch to SavedObjectClient.resolve (elastic#109617) [APM] Adding lambda icon (elastic#111834) ... # Conflicts: # x-pack/plugins/reporting/public/management/__snapshots__/report_listing.test.tsx.snap
- Loading branch information
Showing
1,714 changed files
with
100,397 additions
and
90,437 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,5 @@ | ||
steps: | ||
- command: .buildkite/scripts/steps/es_snapshots/build.sh | ||
label: Build ES Snapshot | ||
agents: | ||
queue: c2-8 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,12 @@ | ||
steps: | ||
- block: 'Promote' | ||
prompt: "Enter the details for the snapshot you'd like to promote" | ||
if: "build.env('ES_SNAPSHOT_MANIFEST') == null" | ||
# Later, this could be a dropdown dynamically filled with recent builds | ||
fields: | ||
- text: 'ES_SNAPSHOT_MANIFEST' | ||
key: 'ES_SNAPSHOT_MANIFEST' | ||
hint: 'URL pointing to the manifest to promote' | ||
required: true | ||
- label: Promote Snapshot | ||
command: .buildkite/scripts/steps/es_snapshots/promote.sh |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,102 @@ | ||
env: | ||
IGNORE_SHIP_CI_STATS_ERROR: 'true' | ||
steps: | ||
- block: 'Verify' | ||
prompt: "Enter the details for the snapshot you'd like to verify" | ||
if: "build.env('ES_SNAPSHOT_MANIFEST') == null" | ||
# Later, this could be a dropdown dynamically filled with recent builds | ||
fields: | ||
- text: 'ES_SNAPSHOT_MANIFEST' | ||
key: 'ES_SNAPSHOT_MANIFEST' | ||
hint: 'URL pointing to the manifest to promote' | ||
required: true | ||
|
||
- command: .buildkite/scripts/lifecycle/pre_build.sh | ||
label: Pre-Build | ||
|
||
- wait | ||
|
||
- command: .buildkite/scripts/steps/build_kibana.sh | ||
label: Build Kibana Distribution and Plugins | ||
agents: | ||
queue: c2-8 | ||
key: build | ||
if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''" | ||
|
||
- command: .buildkite/scripts/steps/functional/xpack_cigroup.sh | ||
label: 'Default CI Group' | ||
parallelism: 13 | ||
agents: | ||
queue: ci-group-6 | ||
artifact_paths: target/junit/**/*.xml | ||
depends_on: build | ||
key: default-cigroup | ||
retry: | ||
automatic: | ||
- exit_status: '*' | ||
limit: 1 | ||
|
||
- command: CI_GROUP=Docker .buildkite/scripts/steps/functional/xpack_cigroup.sh | ||
label: 'Docker CI Group' | ||
agents: | ||
queue: ci-group-6 | ||
artifact_paths: target/junit/**/*.xml | ||
depends_on: build | ||
key: default-cigroup-docker | ||
retry: | ||
automatic: | ||
- exit_status: '*' | ||
limit: 1 | ||
|
||
- command: .buildkite/scripts/steps/functional/oss_cigroup.sh | ||
label: 'OSS CI Group' | ||
parallelism: 12 | ||
agents: | ||
queue: ci-group-4d | ||
artifact_paths: target/junit/**/*.xml | ||
depends_on: build | ||
key: oss-cigroup | ||
retry: | ||
automatic: | ||
- exit_status: '*' | ||
limit: 1 | ||
|
||
- command: .buildkite/scripts/steps/test/jest_integration.sh | ||
label: 'Jest Integration Tests' | ||
agents: | ||
queue: jest | ||
artifact_paths: target/junit/**/*.xml | ||
key: jest-integration | ||
retry: | ||
automatic: | ||
- exit_status: '*' | ||
limit: 1 | ||
|
||
- command: .buildkite/scripts/steps/test/api_integration.sh | ||
label: 'API Integration Tests' | ||
agents: | ||
queue: jest | ||
artifact_paths: target/junit/**/*.xml | ||
key: api-integration | ||
|
||
- command: .buildkite/scripts/steps/es_snapshots/trigger_promote.sh | ||
label: Trigger promotion | ||
depends_on: | ||
- default-cigroup | ||
- default-cigroup-docker | ||
- oss-cigroup | ||
- jest-integration | ||
- api-integration | ||
|
||
- wait: ~ | ||
continue_on_failure: true | ||
|
||
- plugins: | ||
- junit-annotate#v1.9.0: | ||
artifacts: target/junit/**/*.xml | ||
|
||
- wait: ~ | ||
continue_on_failure: true | ||
|
||
- command: .buildkite/scripts/lifecycle/post_build.sh | ||
label: Post-Build |
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,8 @@ | ||
#!/usr/bin/env bash | ||
|
||
set -euo pipefail | ||
|
||
.buildkite/scripts/bootstrap.sh | ||
.buildkite/scripts/build_kibana.sh | ||
.buildkite/scripts/build_kibana_plugins.sh | ||
.buildkite/scripts/post_build_kibana.sh |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,4 @@ | ||
module.exports = { | ||
BASE_BUCKET_DAILY: 'kibana-ci-es-snapshots-daily', | ||
BASE_BUCKET_PERMANENT: 'kibana-ci-es-snapshots-permanent', | ||
}; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,101 @@ | ||
#!/bin/bash | ||
|
||
set -euo pipefail | ||
|
||
source .buildkite/scripts/common/util.sh | ||
|
||
echo "--- Cloning Elasticsearch and preparing workspace" | ||
|
||
cd .. | ||
destination="$(pwd)/es-build" | ||
rm -rf "$destination" | ||
mkdir -p "$destination" | ||
|
||
mkdir -p elasticsearch && cd elasticsearch | ||
|
||
export ELASTICSEARCH_BRANCH="${ELASTICSEARCH_BRANCH:-$BUILDKITE_BRANCH}" | ||
|
||
if [[ ! -d .git ]]; then | ||
git init | ||
git remote add origin https://github.com/elastic/elasticsearch.git | ||
fi | ||
git fetch origin --depth 1 "$ELASTICSEARCH_BRANCH" | ||
git reset --hard FETCH_HEAD | ||
|
||
ELASTICSEARCH_GIT_COMMIT="$(git rev-parse HEAD)" | ||
export ELASTICSEARCH_GIT_COMMIT | ||
|
||
ELASTICSEARCH_GIT_COMMIT_SHORT="$(git rev-parse --short HEAD)" | ||
export ELASTICSEARCH_GIT_COMMIT_SHORT | ||
|
||
# These turn off automation in the Elasticsearch repo | ||
export BUILD_NUMBER="" | ||
export JENKINS_URL="" | ||
export BUILD_URL="" | ||
export JOB_NAME="" | ||
export NODE_NAME="" | ||
export DOCKER_BUILDKIT="" | ||
|
||
# Reads the ES_BUILD_JAVA env var out of .ci/java-versions.properties and exports it | ||
export "$(grep '^ES_BUILD_JAVA' .ci/java-versions.properties | xargs)" | ||
|
||
export PATH="$HOME/.java/$ES_BUILD_JAVA/bin:$PATH" | ||
export JAVA_HOME="$HOME/.java/$ES_BUILD_JAVA" | ||
|
||
# The Elasticsearch Dockerfile needs to be built with root privileges, but Docker on our servers is running using a non-root user | ||
# So, let's use docker-in-docker to temporarily create a privileged docker daemon to run `docker build` on | ||
# We have to do this, because there's no `docker build --privileged` or similar | ||
|
||
echo "--- Setting up Docker-in-Docker for Elasticsearch" | ||
|
||
docker rm -f dind || true # If there's an old daemon running that somehow didn't get cleaned up, lets remove it first | ||
CERTS_DIR="$HOME/dind-certs" | ||
rm -rf "$CERTS_DIR" | ||
docker run -d --rm --privileged --name dind --userns host -p 2377:2376 -e DOCKER_TLS_CERTDIR=/certs -v "$CERTS_DIR":/certs docker:dind | ||
|
||
trap "docker rm -f dind" EXIT | ||
|
||
export DOCKER_TLS_VERIFY=true | ||
export DOCKER_CERT_PATH="$CERTS_DIR/client" | ||
export DOCKER_TLS_CERTDIR="$CERTS_DIR" | ||
export DOCKER_HOST=localhost:2377 | ||
|
||
echo "--- Build Elasticsearch" | ||
./gradlew -Dbuild.docker=true assemble --parallel | ||
|
||
echo "--- Create distribution archives" | ||
find distribution -type f \( -name 'elasticsearch-*-*-*-*.tar.gz' -o -name 'elasticsearch-*-*-*-*.zip' \) -not -path '*no-jdk*' -not -path '*build-context*' -exec cp {} "$destination" \; | ||
|
||
ls -alh "$destination" | ||
|
||
echo "--- Create docker image archives" | ||
docker images "docker.elastic.co/elasticsearch/elasticsearch" | ||
docker images "docker.elastic.co/elasticsearch/elasticsearch" --format "{{.Tag}}" | xargs -n1 echo 'docker save docker.elastic.co/elasticsearch/elasticsearch:${0} | gzip > ../es-build/elasticsearch-${0}-docker-image.tar.gz' | ||
docker images "docker.elastic.co/elasticsearch/elasticsearch" --format "{{.Tag}}" | xargs -n1 bash -c 'docker save docker.elastic.co/elasticsearch/elasticsearch:${0} | gzip > ../es-build/elasticsearch-${0}-docker-image.tar.gz' | ||
|
||
echo "--- Create checksums for snapshot files" | ||
cd "$destination" | ||
find ./* -exec bash -c "shasum -a 512 {} > {}.sha512" \; | ||
|
||
cd "$BUILDKITE_BUILD_CHECKOUT_PATH" | ||
node "$(dirname "${0}")/create_manifest.js" "$destination" | ||
|
||
ES_SNAPSHOT_MANIFEST="$(buildkite-agent meta-data get ES_SNAPSHOT_MANIFEST)" | ||
|
||
cat << EOF | buildkite-agent annotate --style "info" | ||
- \`ELASTICSEARCH_BRANCH\` - \`$ELASTICSEARCH_BRANCH\` | ||
- \`ELASTICSEARCH_GIT_COMMIT\` - \`$ELASTICSEARCH_GIT_COMMIT\` | ||
- \`ES_SNAPSHOT_MANIFEST\` - \`$ES_SNAPSHOT_MANIFEST\` | ||
- \`ES_SNAPSHOT_VERSION\` - \`$(buildkite-agent meta-data get ES_SNAPSHOT_VERSION)\` | ||
- \`ES_SNAPSHOT_ID\` - \`$(buildkite-agent meta-data get ES_SNAPSHOT_ID)\` | ||
EOF | ||
|
||
cat << EOF | buildkite-agent pipeline upload | ||
steps: | ||
- trigger: 'kibana-elasticsearch-snapshot-verify' | ||
async: true | ||
build: | ||
env: | ||
ES_SNAPSHOT_MANIFEST: '$ES_SNAPSHOT_MANIFEST' | ||
branch: '$BUILDKITE_BRANCH' | ||
EOF |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,90 @@ | ||
const fs = require('fs'); | ||
const { execSync } = require('child_process'); | ||
const { BASE_BUCKET_DAILY } = require('./bucket_config.js'); | ||
|
||
(async () => { | ||
console.log('--- Create ES Snapshot Manifest'); | ||
|
||
const destination = process.argv[2] || __dirname + '/test'; | ||
|
||
const ES_BRANCH = process.env.ELASTICSEARCH_BRANCH; | ||
const GIT_COMMIT = process.env.ELASTICSEARCH_GIT_COMMIT; | ||
const GIT_COMMIT_SHORT = process.env.ELASTICSEARCH_GIT_COMMIT_SHORT; | ||
|
||
let VERSION = ''; | ||
let SNAPSHOT_ID = ''; | ||
let DESTINATION = ''; | ||
|
||
const now = new Date(); | ||
|
||
// format: yyyyMMdd-HHmmss | ||
const date = [ | ||
now.getFullYear(), | ||
(now.getMonth() + 1).toString().padStart(2, '0'), | ||
now.getDate().toString().padStart(2, '0'), | ||
'-', | ||
now.getHours().toString().padStart(2, '0'), | ||
now.getMinutes().toString().padStart(2, '0'), | ||
now.getSeconds().toString().padStart(2, '0'), | ||
].join(''); | ||
|
||
try { | ||
const files = fs.readdirSync(destination); | ||
const manifestEntries = files | ||
.filter((filename) => !filename.match(/.sha512$/)) | ||
.filter((filename) => !filename.match(/.json$/)) | ||
.map((filename) => { | ||
const parts = filename.replace('elasticsearch-oss', 'oss').split('-'); | ||
|
||
VERSION = VERSION || parts[1]; | ||
SNAPSHOT_ID = SNAPSHOT_ID || `${date}_${GIT_COMMIT_SHORT}`; | ||
DESTINATION = DESTINATION || `${VERSION}/archives/${SNAPSHOT_ID}`; | ||
|
||
return { | ||
filename: filename, | ||
checksum: filename + '.sha512', | ||
url: `https://storage.googleapis.com/${BASE_BUCKET_DAILY}/${DESTINATION}/${filename}`, | ||
version: parts[1], | ||
platform: parts[3], | ||
architecture: parts[4].split('.')[0], | ||
license: parts[0] == 'oss' ? 'oss' : 'default', | ||
}; | ||
}); | ||
|
||
const manifest = { | ||
id: SNAPSHOT_ID, | ||
bucket: `${BASE_BUCKET_DAILY}/${DESTINATION}`.toString(), | ||
branch: ES_BRANCH, | ||
sha: GIT_COMMIT, | ||
sha_short: GIT_COMMIT_SHORT, | ||
version: VERSION, | ||
generated: now.toISOString(), | ||
archives: manifestEntries, | ||
}; | ||
|
||
const manifestJSON = JSON.stringify(manifest, null, 2); | ||
fs.writeFileSync(`${destination}/manifest.json`, manifestJSON); | ||
|
||
console.log('Manifest:', manifestJSON); | ||
|
||
execSync( | ||
` | ||
set -euo pipefail | ||
echo '--- Upload files to GCS' | ||
cd "${destination}" | ||
gsutil -m cp -r *.* gs://${BASE_BUCKET_DAILY}/${DESTINATION} | ||
cp manifest.json manifest-latest.json | ||
gsutil cp manifest-latest.json gs://${BASE_BUCKET_DAILY}/${VERSION} | ||
buildkite-agent meta-data set ES_SNAPSHOT_MANIFEST 'https://storage.googleapis.com/${BASE_BUCKET_DAILY}/${DESTINATION}/manifest.json' | ||
buildkite-agent meta-data set ES_SNAPSHOT_VERSION '${VERSION}' | ||
buildkite-agent meta-data set ES_SNAPSHOT_ID '${SNAPSHOT_ID}' | ||
`, | ||
{ shell: '/bin/bash' } | ||
); | ||
} catch (ex) { | ||
console.error(ex); | ||
process.exit(1); | ||
} | ||
})(); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,13 @@ | ||
#!/bin/bash | ||
|
||
set -euo pipefail | ||
|
||
export ES_SNAPSHOT_MANIFEST="${ES_SNAPSHOT_MANIFEST:-"$(buildkite-agent meta-data get ES_SNAPSHOT_MANIFEST)"}" | ||
|
||
cat << EOF | buildkite-agent annotate --style "info" | ||
This promotion is for the following snapshot manifest: | ||
$ES_SNAPSHOT_MANIFEST | ||
EOF | ||
|
||
node "$(dirname "${0}")/promote_manifest.js" "$ES_SNAPSHOT_MANIFEST" |
Oops, something went wrong.