diff --git a/.buildkite/deploy/docker/docker-compose.yml b/.buildkite/deploy/docker/docker-compose.yml new file mode 100644 index 000000000000..c27417158a3d --- /dev/null +++ b/.buildkite/deploy/docker/docker-compose.yml @@ -0,0 +1,23 @@ +version: '2.3' +services: + # This is a proxy used to block beats until all services are healthy. + # See: https://github.com/docker/compose/issues/4369 + proxy_dep: + image: busybox + depends_on: + localstack: { condition: service_healthy } + + localstack: + container_name: "${localstack_integration_test_container}" + image: localstack/localstack:3.1.0 # Latest stable release + ports: + - "127.0.0.1:4566:4566" # LocalStack Gateway + environment: + - DEBUG=1 + - DOCKER_HOST=unix:///var/run/docker.sock + - LOCALSTACK_HOST=localhost + - S3_HOSTNAME=localhost + - PROVIDER_OVERRIDE_S3=asf + volumes: + - "${LOCALSTACK_VOLUME_DIR:-./volume}:/var/lib/localstack" + - "/var/run/docker.sock:/var/run/docker.sock" diff --git a/.buildkite/libbeat/pipeline.libbeat.yml b/.buildkite/libbeat/pipeline.libbeat.yml index 91c93716aab8..3d3247b9da98 100644 --- a/.buildkite/libbeat/pipeline.libbeat.yml +++ b/.buildkite/libbeat/pipeline.libbeat.yml @@ -17,7 +17,33 @@ env: TEST_COVERAGE: "true" steps: - - group: "Libbeat Mandatory Tests" + - group: "libbeat Check/Update" + key: "libbeat-check-update" + steps: + - label: "Run check/update" + command: | + make -C libbeat check update + make check-no-changes + retry: + automatic: + - limit: 3 + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest" + cpu: "4000m" + memory: "8Gi" + useCustomGlobalHooks: true + notify: + - github_commit_status: + context: "libbeat: check/update" + + - wait: ~ + # with PRs, we want to run mandatory tests only if check/update step succeed + # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests + # this allows building DRA artifacts even if there is flakiness in check/update step + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: "libbeat-check-update" + + - group: "Mandatory Tests" key: "mandatory-tests" steps: diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 6ce329d35a37..dc7a61feea19 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -378,7 +378,7 @@ startCloudTestEnv() { local dir=$1 withAWS echo "--- Run docker-compose services for emulated cloud env" - docker-compose -f .ci/jobs/docker-compose.yml up -d #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK + docker-compose -f .buildkite/deploy/docker/docker-compose.yml up -d with_Terraform terraformInit "$dir" export TF_VAR_BRANCH=$(echo "${BUILDKITE_BRANCH}" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9-]/-/g') @@ -440,10 +440,10 @@ installNodeJsDependencies() { teardown() { # Teardown resources after using them echo "---Terraform Cleanup" - .ci/scripts/terraform-cleanup.sh "${MODULE_DIR}" #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK + .buildkite/scripts/terraform-cleanup.sh "${MODULE_DIR}" echo "---Docker Compose Cleanup" - docker-compose -f .ci/jobs/docker-compose.yml down -v #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK + docker-compose -f .buildkite/deploy/docker/docker-compose.yml down -v } unset_secrets () { diff --git a/.buildkite/scripts/setup_cloud_env.sh b/.buildkite/scripts/setup_cloud_env.sh index 3c23d71573f1..a13a8446dc71 100644 --- a/.buildkite/scripts/setup_cloud_env.sh +++ b/.buildkite/scripts/setup_cloud_env.sh @@ -11,7 +11,7 @@ teardown() { tf_cleanup "${MODULE_DIR}" #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK echo "~~~ Docker Compose Cleanup" - docker-compose -f .ci/jobs/docker-compose.yml down -v #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK + docker-compose -f .buildkite/deploy/docker/docker-compose.yml down -v } tf_cleanup() { @@ -38,7 +38,7 @@ export TEST_TAGS="${TEST_TAGS:+$TEST_TAGS,}aws" set -o xtrace echo "~~~ Run docker-compose services for emulated cloud env" -docker-compose -f .ci/jobs/docker-compose.yml up -d #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK +docker-compose -f .buildkite/deploy/docker/docker-compose.yml up -d echo "~~~ Initialize TF cloud resources" cd "$MODULE_DIR" export TF_VAR_BRANCH=$(echo "${BUILDKITE_BRANCH}" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9-]/-/g') diff --git a/.ci/scripts/terraform-cleanup.sh b/.buildkite/scripts/terraform-cleanup.sh old mode 100755 new mode 100644 similarity index 100% rename from .ci/scripts/terraform-cleanup.sh rename to .buildkite/scripts/terraform-cleanup.sh diff --git a/.buildkite/x-pack/pipeline.xpack.libbeat.yml b/.buildkite/x-pack/pipeline.xpack.libbeat.yml index e48433b0e058..6cb1825b7a1e 100644 --- a/.buildkite/x-pack/pipeline.xpack.libbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.libbeat.yml @@ -19,6 +19,32 @@ env: TEST_COVERAGE: "true" steps: + - group: "x-pack/libbeat Check/Update" + key: "x-pack-libbeat-check-update" + steps: + - label: "Run check/update" + command: | + make -C x-pack/libbeat check update + make check-no-changes + retry: + automatic: + - limit: 3 + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest" + cpu: "4000m" + memory: "8Gi" + useCustomGlobalHooks: true + notify: + - github_commit_status: + context: "x-pack/libbeat: check/update" + + - wait: ~ + # with PRs, we want to run mandatory tests only if check/update step succeed + # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests + # this allows building DRA artifacts even if there is flakiness in check/update step + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: "x-pack-libbeat-check-update" + - group: "x-pack/libbeat Mandatory Tests" key: "x-pack-libbeat-mandatory-tests" diff --git a/.github/workflows/check-libbeat.yml b/.github/workflows/check-libbeat.yml deleted file mode 100644 index 38b04932a86b..000000000000 --- a/.github/workflows/check-libbeat.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: check-libbeat - -on: - pull_request: - paths: - - '.github/workflows/check-packetbeat.yml' - - 'libbeat/**' - - 'x-pack/libbeat/**' - -env: - BEAT_MODULE: 'libbeat' - -jobs: - check: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v3 - with: - go-version-file: .go-version - - name: Update package lists - run: sudo apt-get update - - name: Install libpcap-dev - run: sudo apt-get install -y libpcap-dev - - name: Run check/update - run: | - go install github.com/magefile/mage - make -C ${{ env.BEAT_MODULE }} check update - make check-no-changes