Merge branch 'main' into bump-apimachinery-v0-31-3

.buildkite/README.md

@@ -23,3 +23,7 @@ After a successful build, the pipeline publishes the generated artifacts to the
 ## Pipeline Configuration
 
 To view the pipeline and its configuration, click [here](https://buildkite.com/elastic/cloudbeat).
+
+## Notifications
+
+The pipeline is [configured](https://buildkite.com/organizations/elastic/services/68636/edit) to send Slack notifications to the `#cloud-sec-ci` channel. Additionally, it includes a custom notification [script](./scripts/notify.sh) that pings specific users in the event of a build failure.

.buildkite/pipeline.yml

@@ -34,3 +34,9 @@ steps:
     command: "./.buildkite/scripts/publish.sh"
     env:
       WORKFLOW: "staging"
+
+  - wait: ~
+    continue_on_failure: true
+
+  - label: "Send Slack Notification"
+    command: ".buildkite/scripts/notify.sh | buildkite-agent pipeline upload"

.buildkite/scripts/notify.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# This script creates a Buildkite notification step and pings the user who triggered the build.
+# It assumes the existence of the GitHub user's email.
+# If the email doesn't exist, the script attempts to extract it from the build message.
+# If no email is found, the default option will be used.
+
+set -euo pipefail
+
+default_team="cloudbeat-eng-team"
+# Check if BUILDKITE_BUILD_CREATOR_EMAIL is set, defaulting to user not found
+build_creator="${BUILDKITE_BUILD_CREATOR_EMAIL:-$default_team}"
+build_message="${BUILDKITE_MESSAGE:-}"
+slack_channel="${SLACK_CHANNEL:-"#cloud-sec-ci"}"
+
+default_group_id=$(vault kv get -field=$default_team secret/ci/elastic-cloudbeat/slack-users)
+user_id=$(vault kv get -field="$build_creator" secret/ci/elastic-cloudbeat/slack-users 2>/dev/null || echo "$default_group_id")
+# If the Slack user is the default one, try to extract email from the BUILD_MESSAGE
+if [[ "$user_id" == "$default_group_id" ]]; then
+    # Extract email from BUILDKITE_MESSAGE
+    email=$(echo "$build_message" | grep -oE '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6}' || echo $default_team)
+    user_id=$(vault kv get -field="$email" secret/ci/elastic-cloudbeat/slack-users 2>/dev/null || echo "$default_group_id")
+fi
+
+# Output the YAML configuration
+cat <<EOF
+notify:
+  - slack:
+      channels:
+        - "$slack_channel"
+      message: "<$user_id>"
+    if: build.state != "passed"
+EOF

.github/actions/cdr/action.yml

@@ -40,6 +40,26 @@ inputs:
     default: "default"
     required: false
     type: string
+  wiz-client-id:
+    description: "Wiz client ID"
+    default: "default"
+    required: false
+    type: string
+  wiz-client-secret:
+    description: "Wiz client secret"
+    default: "default"
+    required: false
+    type: string
+  wiz-endpoint-url:
+    description: "Wiz endpoint URL"
+    default: "default"
+    required: false
+    type: string
+  wiz-token-url:
+    description: "Wiz token URL"
+    default: "default"
+    required: false
+    type: string
   es-user:
     description: "Elasticsearch user"
     default: "elastic"
@@ -133,6 +153,14 @@ runs:
         echo "::add-mask::$asset_inv_public_ip"
         echo "asset-inv-public-ip=$asset_inv_public_ip" >> "$GITHUB_OUTPUT"
 
+        ec2_wiz_key=$(terraform output -raw ec2_wiz_key)
+        echo "::add-mask::$ec2_wiz_key"
+        echo "ec2-wiz-key=$ec2_wiz_key" >> "$GITHUB_OUTPUT"
+
+        ec2_wiz_public_ip=$(terraform output -raw ec2_wiz_public_ip)
+        echo "::add-mask::$ec2_wiz_public_ip"
+        echo "ec2-wiz-public-ip=$ec2_wiz_public_ip" >> "$GITHUB_OUTPUT"
+
     - name: Install AWS Cloudtrail integration
       id: cloudtrail-integration
       if: ${{ !cancelled() && steps.deploy-cdr-infra.outcome  == 'success' }}
@@ -215,6 +243,35 @@ runs:
         cmd="chmod +x $scriptname && ./$scriptname"
         ../remote_setup.sh -k "$AUDIT_LOGS_KEY" -s "$src" -h "$AUDIT_LOGS_PUBLIC_IP" -d "~/$scriptname" -c "$cmd"
 
+    - name: Install WIZ integration
+      id: wiz-integration
+      if: ${{ !cancelled() && steps.deploy-cdr-infra.outcome  == 'success' }}
+      working-directory: tests/integrations_setup
+      shell: bash
+      env:
+        WIZ_CLIENT_ID: ${{ inputs.wiz-client-id }}
+        WIZ_CLIENT_SECRET: ${{ inputs.wiz-client-secret }}
+        WIZ_URL: ${{ inputs.wiz-endpoint-url }}
+        WIZ_TOKEN_URL: ${{ inputs.wiz-token-url }}
+        ES_USER: ${{ inputs.es-user }}
+        ES_PASSWORD: ${{ inputs.es-password }}
+        KIBANA_URL: ${{ inputs.kibana-url }}
+      run: |
+        poetry run python ./install_wiz_integration.py
+
+    - name: Deploy WIZ agent
+      if: ${{ !cancelled() && steps.deploy-cdr-infra.outcome  == 'success' && steps.wiz-integration.outcome == 'success' }}
+      working-directory: deploy/test-environments/cdr
+      shell: bash
+      env:
+        WIZ_KEY: ${{ steps.generate-data.outputs.ec2-wiz-key }}
+        WIZ_PUBLIC_IP: ${{ steps.generate-data.outputs.ec2-wiz-public-ip }}
+      run: |
+        scriptname="wiz.sh"
+        src="../../../tests/integrations_setup/$scriptname"
+        cmd="chmod +x $scriptname && ./$scriptname"
+        ../remote_setup.sh -k "$WIZ_KEY" -s "$src" -h "$WIZ_PUBLIC_IP" -d "~/$scriptname" -c "$cmd"
+
     - name: Check Asset Inventory supported version
       id: asset-inventory-version-check
       if: ${{ !cancelled() && steps.deploy-cdr-infra.outcome  == 'success' }}
@@ -289,5 +346,6 @@ runs:
         ACTIVITY_LOGS_KEY: ${{ steps.generate-data.outputs.az-vm-activity-logs-key }}
         AUDIT_LOGS_KEY: ${{ steps.generate-data.outputs.gcp-audit-logs-key }}
         EC2_ASSET_INV_KEY: ${{ steps.generate-data.outputs.ec2-asset-inv-key }}
+        EC2_WIZ_KEY: ${{ steps.generate-data.outputs.ec2-wiz-key }}
       run: |
         ./manage_infrastructure.sh "cdr" "upload-state"

.github/workflows/bump-version.yml

@@ -19,14 +19,14 @@ jobs:
       - name: Checkout Cloudbeat Repo
         uses: actions/checkout@v4
         with:
+          ref: ${{ github.ref_name }}
           token: ${{ secrets.CLOUDSEC_MACHINE_TOKEN }}
 
       - name: Setup Cloudbeat Versions
         run: |
           current=$(grep defaultBeatVersion version/version.go | cut -f2 -d "\"")
           IFS='.' read -r major minor patch <<< "$current"
-          ((minor++))
-          next="$major.$minor.$patch"
+          next="$major.$((minor+1)).$patch"
           echo "current: $current"
           echo "next: $next"
           echo "CURRENT_CLOUDBEAT_VERSION=$current" >> $GITHUB_ENV
@@ -39,6 +39,8 @@ jobs:
 
       - name: Bump Cloudbeat
         # bump_cloudbeat.sh will create multiple PRs with different HEAD branches
+        env:
+          GIT_BASE_BRANCH: ${{ github.ref_name }}
         run: scripts/bump_cloudbeat.sh
 
       - name: Bump Cloud Security Posture Integration

.github/workflows/ci.yml

@@ -355,7 +355,7 @@ jobs:
 
       - name: Publish allure report
         if: ${{ always() }}
-        uses: andrcuns/allure-publish-action@v2.8.0
+        uses: andrcuns/allure-publish-action@v2.9.0
         env:
           GITHUB_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           AWS_REGION: ${{ env.AWS_REGION }}

.github/workflows/eks-ci.yml

@@ -241,7 +241,7 @@ jobs:
 
       - name: Publish allure report
         if: always()
-        uses: andrcuns/allure-publish-action@v2.8.0
+        uses: andrcuns/allure-publish-action@v2.9.0
         env:
           GITHUB_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           AWS_REGION: eu-west-1

.github/workflows/sync-internal-cloudbeat-version.yml

@@ -0,0 +1,24 @@
+name: Sync Cloudbeat Versions on release
+# Makes sure all cloudbeat versions are updated with release branches
+on:
+  push:
+    branches:
+      # matches branches pushed by elasticmachine, e.g. "update-version-next-8.15.3"
+      - 'update-version-next-*'
+
+jobs:
+  synchronize-versions:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the branch
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.ref }}
+      - name: Synchronize versions using a script
+        run: |
+          ./scripts/sync_internal_cloudbeat_version.sh
+      - name: Push changes
+        env:
+          GITHUB_TOKEN: ${{ secrets.CLOUDSEC_MACHINE_TOKEN }}
+        run:
+          git push origin

.github/workflows/test-environment.yml

@@ -30,13 +30,13 @@ on:
         description: "Provide the full Docker image path to override the default image (e.g. for testing BC/SNAPSHOT)"
         type: string
       run-sanity-tests:
-          description: "Run sanity tests after provision"
-          default: false
-          type: boolean
+        description: "Run sanity tests after provision"
+        default: false
+        type: boolean
       run-ui-sanity-tests:
-          description: "Run UI sanity tests after provision"
-          default: false
-          type: boolean
+        description: "Run UI sanity tests after provision"
+        default: false
+        type: boolean
       kibana_ref:
         description: "Kibana branch, tag, or commit SHA to check out the UI sanity tests from"
         required: false
@@ -81,9 +81,9 @@ on:
         default: false
         type: boolean
       run-ui-sanity-tests:
-          description: "Run UI sanity tests after provision"
-          default: false
-          type: boolean
+        description: "Run UI sanity tests after provision"
+        default: false
+        type: boolean
       kibana_ref:
         description: "Kibana branch, tag, or commit SHA to check out the UI sanity tests from"
         required: false
@@ -134,15 +134,15 @@ jobs:
       TF_VAR_ess_region: ${{ inputs.ess-region }}
       DEPLOYMENT_NAME: ${{ inputs.deployment_name }}
       TF_VAR_serverless_mode: ${{ inputs.serverless_mode }}
-      TEST_AGENTLESS: false # Change to true when creating an Agentless policy
+      TEST_AGENTLESS: true
       S3_BASE_BUCKET: "s3://tf-state-bucket-test-infra"
       S3_BUCKET_URL: "https://s3.console.aws.amazon.com/s3/buckets/tf-state-bucket-test-infra"
       DOCKER_IMAGE_OVERRIDE: ${{ inputs.docker-image-override }}
       CNVM_STACK_NAME: "${{ inputs.deployment_name }}-cnvm-sanity-test-stack"
     # Add "id-token" with the intended permissions.
     permissions:
-      contents: 'read'
-      id-token: 'write'
+      contents: "read"
+      id-token: "write"
     outputs:
       deploy-s3-bucket: ${{ steps.upload-state.outputs.s3-bucket-folder }}
       aws-cnvm-stack-name: ${{ steps.upload-state.outputs.aws-cnvm-stack }}
@@ -230,7 +230,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.9'
+          python-version: "3.9"
 
       - name: Install Poetry
         run: |
@@ -273,7 +273,7 @@ jobs:
         uses: ./.github/actions/elk-stack
         with:
           deployment-name: ${{ env.DEPLOYMENT_NAME }}
-          serverless-mode: ${{ env.TEST_AGENTLESS }}
+          serverless-mode: ${{ env.TF_VAR_serverless_mode }}
           elk-stack-version: ${{ env.STACK_VERSION }}
           ess-region: ${{ env.TF_VAR_ess_region }}
           ec-api-key: ${{ env.TF_VAR_ec_api_key }}
@@ -333,6 +333,10 @@ jobs:
           aws-cloudtrail-s3-bucket: ${{ secrets.CLOUDTRAIL_S3 }}
           azure-eventhub-connection-string: ${{ secrets.AZURE_EVENTHUB_CONNECTION_STRING }}
           azure-storage-account-key: ${{ secrets.AZURE_STORAGE_ACCOUNT_KEY }}
+          wiz-client-id: ${{ secrets.WIZ_CLIENT_ID }}
+          wiz-client-secret: ${{ secrets.WIZ_CLIENT_SECRET }}
+          wiz-endpoint-url: ${{ secrets.WIZ_ENDPOINT_URL }}
+          wiz-token-url: ${{ secrets.WIZ_TOKEN_URL }}
           env-s3-bucket: "${{ env.S3_BASE_BUCKET }}/${{ env.DEPLOYMENT_NAME }}_${{ env.TF_STATE_FOLDER }}"
           es-user: ${{ steps.elk-stack.outputs.es-user }}
           es-password: ${{ steps.elk-stack.outputs.es-password }}

.github/workflows/updatecli.yml

@@ -55,7 +55,7 @@ jobs:
       - name: Init Hermit
         run: ./bin/hermit env -r >> $GITHUB_ENV
       - name: Install Updatecli in the runner
-        uses: updatecli/updatecli-action@704a64517239e0993c5e3bf6749a063b8f950d9f # v0.76.1
+        uses: updatecli/updatecli-action@cb631ef5547ed05db3db64bb2ad42a6cc36e3097 # v0.76.1
       - name: Run Updatecli in Apply mode
         run: updatecli apply --config .ci/updatecli/updatecli.d/update-${{ matrix.pipeline-name }}.yml --values .ci/updatecli/values.yml
         env:
@@ -85,7 +85,7 @@ jobs:
 
           echo "GIT_BRANCH=$branchName" >> $GITHUB_ENV
       - name: Install Updatecli in the runner
-        uses: updatecli/updatecli-action@704a64517239e0993c5e3bf6749a063b8f950d9f # v0.76.1
+        uses: updatecli/updatecli-action@cb631ef5547ed05db3db64bb2ad42a6cc36e3097 # v0.76.1
       - name: Run Updatecli in Apply mode
         run: updatecli apply --config .ci/updatecli/updatecli.d/update-${{ matrix.pipeline-name }}.yml --values .ci/updatecli/values.yml
         env:

.github/workflows/weekly-enviroment.yml

@@ -83,7 +83,7 @@ jobs:
 
       # Once https://github.com/slackapi/slack-github-action/issues/84 will be resolved we can push the payload to a different file
       - name: Send custom JSON data to Slack workflow
-        uses: slackapi/slack-github-action@v1.27.0
+        uses: slackapi/slack-github-action@v2.0.0
         with:
           payload: |
             {

.mergify.yml

@@ -339,3 +339,16 @@ pull_request_rules:
         labels:
           - "backport"
         title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}"
+  - name: backport patches to 8.17 branch
+    conditions:
+      - merged
+      - label=backport-8.17
+    actions:
+      backport:
+        assignees:
+          - "{{ author }}"
+        branches:
+          - "8.17"
+        labels:
+          - "backport"
+        title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}"

.pre-commit-config.yaml

@@ -42,7 +42,7 @@ repos:
         files: ^(.ci/scripts|.github/actions|tests)/
 
   - repo: https://github.com/pycqa/pylint
-    rev: v3.3.1
+    rev: v3.3.2
     hooks:
       - id: pylint
         args: [
@@ -77,7 +77,7 @@ repos:
         files: deploy/cloudformation/.*yml
 
   - repo: https://github.com/aws-cloudformation/cfn-lint
-    rev: v1.19.0
+    rev: v1.20.1
     hooks:
       - id: cfn-python-lint
         files: deploy/cloudformation/.*.yml
@@ -170,7 +170,7 @@ repos:
         files: (\.rego)$
 
   - repo: https://github.com/igorshubovych/markdownlint-cli
-    rev: v0.42.0
+    rev: v0.43.0
     hooks:
       - id: markdownlint
         args: [ '--disable',

bin/aws

@@ -1 +1 @@
-.awscli-2.21.3.pkg
+.awscli-2.22.7.pkg

bin/aws-iam-authenticator

@@ -1 +1 @@
-.aws-iam-authenticator-0.6.27.pkg
+.aws-iam-authenticator-0.6.28.pkg

bin/aws_completer

@@ -1 +1 @@
-.awscli-2.21.3.pkg
+.awscli-2.22.7.pkg

bin/bq

@@ -1 +1 @@
-.gcloud-501.0.0.pkg
+.gcloud-502.0.0.pkg

bin/docker-credential-gcloud

@@ -1 +1 @@
-.gcloud-501.0.0.pkg
+.gcloud-502.0.0.pkg

bin/elastic-package

@@ -1 +1 @@
-.elastic-package-0.107.0.pkg
+.elastic-package-0.107.2.pkg

bin/gcloud

@@ -1 +1 @@
-.gcloud-501.0.0.pkg
+.gcloud-502.0.0.pkg

bin/gh

@@ -1 +1 @@
-.gh-2.62.0.pkg
+.gh-2.63.0.pkg