Merge branch 'main' into ml-aiops-api-license-check

elastic · Jun 30, 2022 · e511d8b · e511d8b
2 parents c205a25 + 4179903
commit e511d8b
Show file tree

Hide file tree

Showing 654 changed files with 91,129 additions and 82,063 deletions.
diff --git a/.buildkite/scripts/build_kibana.sh b/.buildkite/scripts/build_kibana.sh
@@ -15,6 +15,28 @@ else
   node scripts/build
 fi
 
+if is_pr_with_label "ci:build-cloud-image"; then
+  echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co
+  node scripts/build \
+  --skip-initialize \
+  --skip-generic-folders \
+  --skip-platform-folders \
+  --skip-archives \
+  --docker-images \
+  --docker-tag-qualifier="$GIT_COMMIT" \
+  --docker-push \
+  --skip-docker-ubi \
+  --skip-docker-ubuntu \
+  --skip-docker-contexts
+  docker logout docker.elastic.co
+
+  CLOUD_IMAGE=$(docker images --format "{{.Repository}}:{{.Tag}}" docker.elastic.co/kibana-ci/kibana-cloud)
+  cat << EOF | buildkite-agent annotate --style "info" --context kibana-cloud-image
+
+  Kibana cloud image: \`$CLOUD_IMAGE\`
+EOF
+fi
+
 echo "--- Archive Kibana Distribution"
 linuxBuild="$(find "$KIBANA_DIR/target" -name 'kibana-*-linux-x86_64.tar.gz')"
 installDir="$KIBANA_DIR/install/kibana"

diff --git a/.buildkite/scripts/lifecycle/post_command.sh b/.buildkite/scripts/lifecycle/post_command.sh
@@ -14,6 +14,7 @@ if [[ "$IS_TEST_EXECUTION_STEP" == "true" ]]; then
   buildkite-agent artifact upload 'target/kibana-coverage/functional/**/*'
   buildkite-agent artifact upload 'target/kibana-*'
   buildkite-agent artifact upload 'target/kibana-security-solution/**/*.png'
+  buildkite-agent artifact upload 'target/kibana-fleet/**/*.png'
   buildkite-agent artifact upload 'target/test-metrics/*'
   buildkite-agent artifact upload 'target/test-suites-ci-plan.json'
   buildkite-agent artifact upload 'test/**/screenshots/diff/*.png'

diff --git a/.buildkite/scripts/steps/cloud/build_and_deploy.sh b/.buildkite/scripts/steps/cloud/build_and_deploy.sh
@@ -10,40 +10,48 @@ export KBN_NP_PLUGINS_BUILT=true
 
 VERSION="$(jq -r '.version' package.json)-SNAPSHOT"
 
-echo "--- Download kibana distribution"
+echo "--- Download Kibana Distribution"
 
 mkdir -p ./target
 buildkite-agent artifact download "kibana-$VERSION-linux-x86_64.tar.gz" ./target --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
 
-echo "--- Build and push Kibana Cloud Distribution"
-
-echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co
-trap 'docker logout docker.elastic.co' EXIT
-
-node scripts/build \
-  --skip-initialize \
-  --skip-generic-folders \
-  --skip-platform-folders \
-  --skip-archives \
-  --docker-images \
-  --docker-tag-qualifier="$GIT_COMMIT" \
-  --docker-push \
-  --skip-docker-ubi \
-  --skip-docker-ubuntu \
-  --skip-docker-contexts
-
+echo "--- Build Cloud Distribution"
 ELASTICSEARCH_MANIFEST_URL="https://storage.googleapis.com/kibana-ci-es-snapshots-daily/$(jq -r '.version' package.json)/manifest-latest-verified.json"
 ELASTICSEARCH_SHA=$(curl -s $ELASTICSEARCH_MANIFEST_URL | jq -r '.sha')
 ELASTICSEARCH_CLOUD_IMAGE="docker.elastic.co/kibana-ci/elasticsearch-cloud:$VERSION-$ELASTICSEARCH_SHA"
 
-CLOUD_IMAGE=$(docker images --format "{{.Repository}}:{{.Tag}}" docker.elastic.co/kibana-ci/kibana-cloud)
+KIBANA_CLOUD_IMAGE="docker.elastic.co/kibana-ci/kibana-cloud:$VERSION-$GIT_COMMIT"
 CLOUD_DEPLOYMENT_NAME="kibana-pr-$BUILDKITE_PULL_REQUEST"
 
+echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co
+set +e
+DISTRIBUTION_EXISTS=$(docker manifest inspect $KIBANA_CLOUD_IMAGE &> /dev/null; echo $?)
+set -e
+
+if  [ $DISTRIBUTION_EXISTS -eq 0 ]; then
+  echo "Distribution already exists, skipping build"
+else
+  node scripts/build \
+    --skip-initialize \
+    --skip-generic-folders \
+    --skip-platform-folders \
+    --skip-archives \
+    --docker-images \
+    --docker-tag-qualifier="$GIT_COMMIT" \
+    --docker-push \
+    --skip-docker-ubi \
+    --skip-docker-ubuntu \
+    --skip-docker-contexts
+fi
+
+docker logout docker.elastic.co
+
+echo "--- Create Deployment"
 CLOUD_DEPLOYMENT_ID=$(ecctl deployment list --output json | jq -r '.deployments[] | select(.name == "'$CLOUD_DEPLOYMENT_NAME'") | .id')
 JSON_FILE=$(mktemp --suffix ".json")
 if [ -z "${CLOUD_DEPLOYMENT_ID}" ]; then
   jq '
-    .resources.kibana[0].plan.kibana.docker_image = "'$CLOUD_IMAGE'" |
+    .resources.kibana[0].plan.kibana.docker_image = "'$KIBANA_CLOUD_IMAGE'" |
     .resources.elasticsearch[0].plan.elasticsearch.docker_image = "'$ELASTICSEARCH_CLOUD_IMAGE'" |
     .name = "'$CLOUD_DEPLOYMENT_NAME'" |
     .resources.kibana[0].plan.kibana.version = "'$VERSION'" |
@@ -52,32 +60,38 @@ if [ -z "${CLOUD_DEPLOYMENT_ID}" ]; then
     .resources.integrations_server[0].plan.integrations_server.version = "'$VERSION'"
     ' .buildkite/scripts/steps/cloud/deploy.json > /tmp/deploy.json
 
-  ecctl deployment create --track --output json --file /tmp/deploy.json &> "$JSON_FILE"
+  echo -n "Creating deployment..."
+  ecctl deployment create --track --output json --file /tmp/deploy.json > "$JSON_FILE"
+  echo "done"
+
   CLOUD_DEPLOYMENT_USERNAME=$(jq --slurp '.[]|select(.resources).resources[] | select(.credentials).credentials.username' "$JSON_FILE")
   CLOUD_DEPLOYMENT_PASSWORD=$(jq --slurp '.[]|select(.resources).resources[] | select(.credentials).credentials.password' "$JSON_FILE")
   CLOUD_DEPLOYMENT_ID=$(jq -r --slurp '.[0].id' "$JSON_FILE")
   CLOUD_DEPLOYMENT_STATUS_MESSAGES=$(jq --slurp '[.[]|select(.resources == null)]' "$JSON_FILE")
 
-  # Enable stack monitoring
-  jq '
-    .settings.observability.metrics.destination.deployment_id = "'$CLOUD_DEPLOYMENT_ID'" |
-    .settings.observability.logging.destination.deployment_id = "'$CLOUD_DEPLOYMENT_ID'"
-    ' .buildkite/scripts/steps/cloud/stack_monitoring.json > /tmp/stack_monitoring.json
-  ecctl deployment update "$CLOUD_DEPLOYMENT_ID" --track --output json --file /tmp/stack_monitoring.json &> "$JSON_FILE"
-
-  # Refresh vault token
+  echo -n "Writing to vault..."
   VAULT_ROLE_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-role-id)"
   VAULT_SECRET_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-secret-id)"
   VAULT_TOKEN=$(retry 5 30 vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID")
   retry 5 30 vault login -no-print "$VAULT_TOKEN"
-
   retry 5 5 vault write "secret/kibana-issues/dev/cloud-deploy/$CLOUD_DEPLOYMENT_NAME" username="$CLOUD_DEPLOYMENT_USERNAME" password="$CLOUD_DEPLOYMENT_PASSWORD"
+
+  echo -n "Enabling Stack Monitoring..."
+  jq '
+    .settings.observability.metrics.destination.deployment_id = "'$CLOUD_DEPLOYMENT_ID'" |
+    .settings.observability.logging.destination.deployment_id = "'$CLOUD_DEPLOYMENT_ID'"
+    ' .buildkite/scripts/steps/cloud/stack_monitoring.json > /tmp/stack_monitoring.json
+  ecctl deployment update "$CLOUD_DEPLOYMENT_ID" --track --output json --file /tmp/stack_monitoring.json > "$JSON_FILE"
+  echo "done"
 else
 ecctl deployment show "$CLOUD_DEPLOYMENT_ID" --generate-update-payload | jq '
-  .resources.kibana[0].plan.kibana.docker_image = "'$CLOUD_IMAGE'" |
+  .resources.kibana[0].plan.kibana.docker_image = "'$KIBANA_CLOUD_IMAGE'" |
   (.. | select(.version? != null).version) = "'$VERSION'"
   ' > /tmp/deploy.json
-  ecctl deployment update "$CLOUD_DEPLOYMENT_ID" --track --output json --file /tmp/deploy.json &> "$JSON_FILE"
+
+  echo -n "Updating deployment..."
+  ecctl deployment update "$CLOUD_DEPLOYMENT_ID" --track --output json --file /tmp/deploy.json > "$JSON_FILE"
+  echo "done"
 fi
 
 CLOUD_DEPLOYMENT_KIBANA_URL=$(ecctl deployment show "$CLOUD_DEPLOYMENT_ID" | jq -r '.resources.kibana[0].info.metadata.aliased_url')
@@ -92,7 +106,7 @@ cat << EOF | buildkite-agent annotate --style "info" --context cloud
 
   Credentials: \`vault read secret/kibana-issues/dev/cloud-deploy/$CLOUD_DEPLOYMENT_NAME\`
 
-  Kibana image: \`$CLOUD_IMAGE\`
+  Kibana image: \`$KIBANA_CLOUD_IMAGE\`
 
   Elasticsearch image: \`$ELASTICSEARCH_CLOUD_IMAGE\`
 EOF

diff --git a/.buildkite/scripts/steps/functional/fleet_cypress.sh b/.buildkite/scripts/steps/functional/fleet_cypress.sh
@@ -2,10 +2,7 @@
 
 set -euo pipefail
 
-source .buildkite/scripts/common/util.sh
-
-.buildkite/scripts/bootstrap.sh
-.buildkite/scripts/download_build_artifacts.sh
+source .buildkite/scripts/steps/functional/common.sh
 
 export JOB=kibana-fleet-cypress
 

diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -363,7 +363,7 @@
 /x-pack/test/functional_with_es_ssl/fixtures/plugins/alerts/ @elastic/response-ops
 /docs/user/alerting/ @elastic/response-ops @elastic/mlr-docs
 /docs/management/connectors/ @elastic/response-ops @elastic/mlr-docs
-#CC# /x-pack/plugins/stack_alerts @elastic/response-ops
+#CC# /x-pack/plugins/stack_alerts/ @elastic/response-ops
 /x-pack/plugins/cases/ @elastic/response-ops
 /x-pack/test/cases_api_integration/ @elastic/response-ops
 /x-pack/test/functional/services/cases/ @elastic/response-ops
@@ -570,8 +570,8 @@
 /x-pack/test/security_solution_endpoint_api_int/ @elastic/security-onboarding-and-lifecycle-mgt
 
 ## Security Solution sub teams - security-telemetry (Data Engineering)
-x-pack/plugins/security_solution/server/usage/ @elastic/security-telemetry
-x-pack/plugins/security_solution/server/lib/telemetry/ @elastic/security-telemetry
+x-pack/plugins/security_solution/server/usage/ @elastic/security-data-analytics
+x-pack/plugins/security_solution/server/lib/telemetry/ @elastic/security-data-analytics
 
 ## Security Solution sub teams - security-engineering-productivity
 x-pack/plugins/security_solution/cypress/ccs_integration @elastic/security-engineering-productivity

diff --git a/api_docs/actions.mdx b/api_docs/actions.mdx
@@ -4,7 +4,7 @@ slug: /kibana-dev-docs/api/actions
 title: "actions"
 image: https://source.unsplash.com/400x175/?github
 summary: API docs for the actions plugin
-date: 2022-06-29
+date: 2022-06-30
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'actions']
 warning: This document is auto-generated and is meant to be viewed inside our experimental, new docs system. Reach out in #docs-engineering for more info.
 ---

diff --git a/api_docs/advanced_settings.mdx b/api_docs/advanced_settings.mdx
@@ -4,7 +4,7 @@ slug: /kibana-dev-docs/api/advancedSettings
 title: "advancedSettings"
 image: https://source.unsplash.com/400x175/?github
 summary: API docs for the advancedSettings plugin
-date: 2022-06-29
+date: 2022-06-30
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'advancedSettings']
 warning: This document is auto-generated and is meant to be viewed inside our experimental, new docs system. Reach out in #docs-engineering for more info.
 ---

diff --git a/api_docs/aiops.mdx b/api_docs/aiops.mdx
@@ -4,7 +4,7 @@ slug: /kibana-dev-docs/api/aiops
 title: "aiops"
 image: https://source.unsplash.com/400x175/?github
 summary: API docs for the aiops plugin
-date: 2022-06-29
+date: 2022-06-30
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'aiops']
 warning: This document is auto-generated and is meant to be viewed inside our experimental, new docs system. Reach out in #docs-engineering for more info.
 ---

diff --git a/api_docs/alerting.mdx b/api_docs/alerting.mdx
@@ -4,7 +4,7 @@ slug: /kibana-dev-docs/api/alerting
 title: "alerting"
 image: https://source.unsplash.com/400x175/?github
 summary: API docs for the alerting plugin
-date: 2022-06-29
+date: 2022-06-30
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'alerting']
 warning: This document is auto-generated and is meant to be viewed inside our experimental, new docs system. Reach out in #docs-engineering for more info.
 ---

diff --git a/api_docs/apm.mdx b/api_docs/apm.mdx
@@ -4,7 +4,7 @@ slug: /kibana-dev-docs/api/apm
 title: "apm"
 image: https://source.unsplash.com/400x175/?github
 summary: API docs for the apm plugin
-date: 2022-06-29
+date: 2022-06-30
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'apm']
 warning: This document is auto-generated and is meant to be viewed inside our experimental, new docs system. Reach out in #docs-engineering for more info.
 ---

diff --git a/api_docs/banners.mdx b/api_docs/banners.mdx
@@ -4,7 +4,7 @@ slug: /kibana-dev-docs/api/banners
 title: "banners"
 image: https://source.unsplash.com/400x175/?github
 summary: API docs for the banners plugin
-date: 2022-06-29
+date: 2022-06-30
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'banners']
 warning: This document is auto-generated and is meant to be viewed inside our experimental, new docs system. Reach out in #docs-engineering for more info.
 ---

diff --git a/api_docs/bfetch.mdx b/api_docs/bfetch.mdx
@@ -4,7 +4,7 @@ slug: /kibana-dev-docs/api/bfetch
 title: "bfetch"
 image: https://source.unsplash.com/400x175/?github
 summary: API docs for the bfetch plugin
-date: 2022-06-29
+date: 2022-06-30
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'bfetch']
 warning: This document is auto-generated and is meant to be viewed inside our experimental, new docs system. Reach out in #docs-engineering for more info.
 ---

diff --git a/api_docs/canvas.mdx b/api_docs/canvas.mdx
@@ -4,7 +4,7 @@ slug: /kibana-dev-docs/api/canvas
 title: "canvas"
 image: https://source.unsplash.com/400x175/?github
 summary: API docs for the canvas plugin
-date: 2022-06-29
+date: 2022-06-30
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'canvas']
 warning: This document is auto-generated and is meant to be viewed inside our experimental, new docs system. Reach out in #docs-engineering for more info.
 ---

diff --git a/api_docs/cases.mdx b/api_docs/cases.mdx
@@ -4,7 +4,7 @@ slug: /kibana-dev-docs/api/cases
 title: "cases"
 image: https://source.unsplash.com/400x175/?github
 summary: API docs for the cases plugin
-date: 2022-06-29
+date: 2022-06-30
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'cases']
 warning: This document is auto-generated and is meant to be viewed inside our experimental, new docs system. Reach out in #docs-engineering for more info.
 ---

diff --git a/api_docs/charts.mdx b/api_docs/charts.mdx
@@ -4,7 +4,7 @@ slug: /kibana-dev-docs/api/charts
 title: "charts"
 image: https://source.unsplash.com/400x175/?github
 summary: API docs for the charts plugin
-date: 2022-06-29
+date: 2022-06-30
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'charts']
 warning: This document is auto-generated and is meant to be viewed inside our experimental, new docs system. Reach out in #docs-engineering for more info.
 ---

diff --git a/api_docs/cloud.mdx b/api_docs/cloud.mdx
@@ -4,7 +4,7 @@ slug: /kibana-dev-docs/api/cloud
 title: "cloud"
 image: https://source.unsplash.com/400x175/?github
 summary: API docs for the cloud plugin
-date: 2022-06-29
+date: 2022-06-30
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'cloud']
 warning: This document is auto-generated and is meant to be viewed inside our experimental, new docs system. Reach out in #docs-engineering for more info.
 ---

diff --git a/api_docs/cloud_security_posture.mdx b/api_docs/cloud_security_posture.mdx
@@ -4,7 +4,7 @@ slug: /kibana-dev-docs/api/cloudSecurityPosture
 title: "cloudSecurityPosture"
 image: https://source.unsplash.com/400x175/?github
 summary: API docs for the cloudSecurityPosture plugin
-date: 2022-06-29
+date: 2022-06-30
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'cloudSecurityPosture']
 warning: This document is auto-generated and is meant to be viewed inside our experimental, new docs system. Reach out in #docs-engineering for more info.
 ---

diff --git a/api_docs/console.mdx b/api_docs/console.mdx
@@ -4,7 +4,7 @@ slug: /kibana-dev-docs/api/console
 title: "console"
 image: https://source.unsplash.com/400x175/?github
 summary: API docs for the console plugin
-date: 2022-06-29
+date: 2022-06-30
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'console']
 warning: This document is auto-generated and is meant to be viewed inside our experimental, new docs system. Reach out in #docs-engineering for more info.
 ---

diff --git a/api_docs/controls.mdx b/api_docs/controls.mdx
@@ -4,7 +4,7 @@ slug: /kibana-dev-docs/api/controls
 title: "controls"
 image: https://source.unsplash.com/400x175/?github
 summary: API docs for the controls plugin
-date: 2022-06-29
+date: 2022-06-30
 tags: ['contributor', 'dev', 'apidocs', 'kibana', 'controls']
 warning: This document is auto-generated and is meant to be viewed inside our experimental, new docs system. Reach out in #docs-engineering for more info.
 ---