k8s

.github/actions/aws-ci/action.yml

@@ -45,7 +45,7 @@ runs:
 
     - name: Wait for cloudbeat to send some events
       shell: bash
-      run: sleep 10
+      run: sleep 20
 
     - name: Check for findings
       working-directory: ./tests
@@ -55,7 +55,7 @@ runs:
       run: poetry run pytest -k "aws" --alluredir=./allure/results/ --clean-alluredir
 
     - name: Upload test results
-      if: always()
+      if: ${{ success() || failure() }}
       uses: actions/upload-artifact@v4
       with:
         name: allure-results-ci-aws

.github/actions/azure-ci/action.yml

@@ -60,7 +60,7 @@ runs:
       run: pkill -15 cloudbeat
 
     - name: Upload test results
-      if: always()
+      if: ${{ success() || failure() }}
       uses: actions/upload-artifact@v4
       with:
         name: allure-results-ci-azure

.github/actions/cnvm-ci/action.yml

@@ -62,7 +62,7 @@ runs:
       run: pkill -15 cloudbeat
 
     - name: Upload test results
-      if: always()
+      if: ${{ success() || failure() }}
       uses: actions/upload-artifact@v4
       with:
         name: allure-results-ci-cnvm

.github/actions/docker-images/action.yml

@@ -0,0 +1,100 @@
+name: 'Docker Images'
+description: 'Build docker images'
+inputs:
+  elk-version:
+    description: 'ELK version'
+    required: true
+  container-image-suffix:
+    description: 'Container image suffix'
+    required: true
+  elastic-agent-docker-image:
+    description: 'Elastic-Agent docker image'
+    required: true
+  elastic-agent-docker-image-tag:
+    description: 'Elastic-Agent docker image tag'
+    required: true
+runs:
+  using: composite
+  steps:
+    - name: Free Disk Space (Ubuntu)
+      uses: jlumbroso/free-disk-space@main
+      with:
+        tool-cache: false
+        android: true
+        dotnet: true
+        haskell: true
+        large-packages: false
+        docker-images: true
+        swap-storage: true
+
+    - name: Init directories
+      shell: bash
+      run: |
+        mkdir -p /tmp/.buildx-cache/
+        mkdir -p /tmp/.buildx-cache-new/
+        mkdir -p /tmp/docker-images/
+
+    - name: Build cloudbeat binary
+      shell: bash
+      run: |
+        mage -v build
+        ldd cloudbeat
+        file cloudbeat
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+
+    - name: Cache docker build cache
+      uses: actions/cache@v4
+      with:
+        path: /tmp/.buildx-cache
+        key: ci-buildx-${{ runner.os }}-${{ runner.arch }}-${{ github.workflow }}
+
+    - name: Cache docker images
+      uses: actions/cache@v4
+      with:
+        path: /tmp/docker-images/
+        key: ci-docker-images-${{ runner.os }}-${{ runner.arch }}-${{ github.workflow }}-${{ inputs.container-image-suffix }}
+
+    - name: Build cloudbeat-docker image
+      uses: docker/build-push-action@v5
+      with:
+        context: .
+        file: ./deploy/Dockerfile
+        push: false
+        tags: cloudbeat:latest
+        cache-from: type=local,src=/tmp/.buildx-cache
+        cache-to: type=local,dest=/tmp/.buildx-cache-new
+        outputs: type=docker,dest=/tmp/docker-images/cloudbeat-${{ inputs.container-image-suffix }}.tar
+
+    - name: Build elastic-agent
+      uses: docker/build-push-action@v5
+      env:
+        GOOS: linux
+        GOARCH: amd64
+      with:
+        context: .
+        file: ./scripts/packaging/docker/elastic-agent/Dockerfile
+        push: false
+        pull: true
+        tags: ${{ inputs.elastic-agent-docker-image }}:${{ inputs.elastic-agent-docker-image-tag }}
+        cache-from: type=local,src=/tmp/.buildx-cache
+        cache-to: type=local,dest=/tmp/.buildx-cache-new
+        outputs: type=docker,dest=/tmp/docker-images/elastic-agent-${{ inputs.container-image-suffix }}.tar
+        build-args: ELASTIC_AGENT_IMAGE=docker.elastic.co/beats/elastic-agent:${{ inputs.elastic-agent-docker-image-tag }}
+
+    - name: Build pytest-docker
+      uses: docker/build-push-action@v5
+      with:
+        context: ./tests/.
+        push: false
+        tags: cloudbeat-test:latest
+        cache-from: type=local,mode=max,src=/tmp/.buildx-cache
+        cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-new
+        outputs: type=docker,dest=/tmp/docker-images/pytest-${{ inputs.container-image-suffix }}.tar
+
+    - name: Rotate cache
+      shell: bash
+      run: |
+        ls -lahR /tmp/ || true
+        [[ -d /tmp/.buildx-cache-new ]] && rm -rf /tmp/.buildx-cache && mv /tmp/.buildx-cache-new /tmp/.buildx-cache

.github/actions/gcp-ci/action.yml

@@ -64,7 +64,7 @@ runs:
       run: pkill -15 cloudbeat
 
     - name: Upload test results
-      if: always()
+      if: ${{ success() || failure() }}
       uses: actions/upload-artifact@v4
       with:
         name: allure-results-ci-gcp

.github/actions/hermit/action.yml

@@ -33,6 +33,16 @@ runs:
         restore-keys: |
           ci-hermit-env-${{ runner.os }}
 
+    - if: ${{ inputs.cache == 'true' }}
+      id: cache-go-deps
+      uses: actions/cache@v4
+      with:
+        path: |
+          ~/go/pkg/
+        key: ci-go-deps-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/go.mod', '**/go.sum') }}
+        restore-keys: |
+          ci-go-deps-${{ runner.os }}-${{ runner.arch }}
+
     - name: Initialize hermit
       shell: bash
       run: ./bin/hermit env --raw >> "$GITHUB_ENV"
@@ -59,5 +69,7 @@ runs:
       shell: bash
       run: pre-commit || true
 
+    - shell: bash
+      run: go mod download
     - shell: bash
       run: go env

.github/actions/k8s-ci/action.yml

@@ -0,0 +1,97 @@
+name: 'K8S CI'
+description: 'K8s integration tests'
+inputs:
+  elk-version:
+    description: 'ELK version'
+    required: true
+  kind-config:
+    description: 'KIND configuration'
+    required: true
+  container-image-suffix:
+    description: 'Container image suffix'
+    required: true
+  elastic-agent-docker-image:
+    description: 'Elastic-Agent docker image'
+    required: true
+  elastic-agent-docker-image-tag:
+    description: 'Elastic-Agent docker image tag'
+    required: true
+
+  test-target:
+    description: 'Test target'
+    required: true
+  test-range:
+    description: 'Test range'
+    required: false
+    default: ''
+  values-file:
+    description: 'Helm values file'
+    required: true
+runs:
+  using: composite
+  steps:
+    - name: Free Disk Space (Ubuntu)
+      uses: jlumbroso/free-disk-space@main
+      with:
+        tool-cache: false
+        android: true
+        dotnet: true
+        haskell: true
+        large-packages: false
+        docker-images: true
+        swap-storage: true
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+
+    - name: Cache docker images
+      uses: actions/cache@v4
+      with:
+        path: /tmp/docker-images/
+        key: ci-docker-images-${{ runner.os }}-${{ runner.arch }}-${{ github.workflow }}-${{ inputs.container-image-suffix }}
+
+    - name: Prepare Kind Cluster for Process Tests
+      if: ${{ contains(inputs.kind-config, 'conf2') }}
+      shell: bash
+      run: |
+        # Workaround: Direct creation of a Kind cluster for config2 fails to start control-plane (kubelet fails)
+        # Creating and deleting a kind-mono cluster as a preparation for configuring Kind for process tests
+        just create-kind-cluster kind-mono
+        just delete-kind-cluster kind-mono
+
+    - name: Create k8s Kind Cluster
+      shell: bash
+      run: just create-kind-cluster ${{ inputs.kind-config }}
+
+    # - name: Load images to kind
+    #   shell: bash
+    #   run: ./.ci/scripts/kind-images.sh '${{ inputs.container-image-suffix }}' '${{ inputs.kind-config }}'
+    - name: Load images to kind
+      shell: bash
+      run: |
+        ls -lahR /tmp/docker-images/
+        kind load image-archive /tmp/docker-images/cloudbeat-${{ inputs.container-image-suffix }}.tar --name='${{ inputs.kind-config }}'
+        kind load image-archive /tmp/docker-images/pytest-${{ inputs.container-image-suffix }}.tar --name='${{ inputs.kind-config }}'
+        kind load image-archive /tmp/docker-images/elastic-agent-${{ inputs.container-image-suffix }}.tar --name='${{ inputs.kind-config }}'
+
+    - name: Deploy tests Helm chart
+      id: deploy_helm
+      shell: bash
+      run: just deploy-tests-helm ${{ inputs.test-target }} ${{ inputs.values-file }} ${{ inputs.test-range }}
+
+    - name: Deploy K8s Objects
+      if: ${{ contains(inputs.test-target, 'object_psp') }}
+      shell: bash
+      run: just apply-k8s-test-objects
+
+    - name: Run Tests
+      id: run_tests
+      shell: bash
+      run: just run-tests ${{ inputs.test-target }} kind-${{ inputs.kind-config }}
+
+    - name: Upload Test Results
+      if: ${{ success() || failure() }}
+      uses: actions/upload-artifact@v4
+      with:
+        name: allure-results-ci-k8s-${{ inputs.test-target }}-${{ inputs.kind-config }}
+        path: tests/allure/results/

.github/workflows/cloudbeat-ci.yml

@@ -9,7 +9,7 @@ on:
 
 env:
   CONTAINER_SUFFIX: ${{ github.run_id }}
-  CI_ELASTIC_AGENT_DOCKER_TAG: "8.13.0-SNAPSHOT"
+  CI_ELASTIC_AGENT_DOCKER_TAG: "8.14.0-SNAPSHOT"
   CI_ELASTIC_AGENT_DOCKER_IMAGE: "704479110758.dkr.ecr.eu-west-2.amazonaws.com/elastic-agent"
 
 concurrency:
@@ -18,6 +18,7 @@ concurrency:
 
 jobs:
   Lint:
+    if: ${{ false }}
     # for more information see .pre-commit-config.yaml
     name: Pre-commit Hooks
     runs-on: ubuntu-latest
@@ -70,6 +71,7 @@ jobs:
         run: just validate-mocks
 
   Build:
+    if: ${{ false }}
     name: Build
     runs-on: ubuntu-20.04
     timeout-minutes: 20

.github/workflows/eks-ci.yml

@@ -24,7 +24,7 @@ env:
   AWS_REGION: eu-west-2
   REPORTS_DIR: tests/allure/results/
   KUBE_NAMESPACE: kube-system
-  CI_ELASTIC_AGENT_DOCKER_TAG: "8.13.0-SNAPSHOT"
+  CI_ELASTIC_AGENT_DOCKER_TAG: "8.14.0-SNAPSHOT"
   CI_ELASTIC_AGENT_DOCKER_IMAGE: "391946104644.dkr.ecr.eu-west-2.amazonaws.com/elastic-agent"
 
 # run only a single job at a time

.github/workflows/k8s-ci.yml

@@ -14,7 +14,7 @@ on:
 
 env:
   CONTAINER_SUFFIX: ${{ github.run_id }}
-  CI_ELASTIC_AGENT_DOCKER_TAG: "8.13.0-SNAPSHOT"
+  CI_ELASTIC_AGENT_DOCKER_TAG: "8.14.0-SNAPSHOT"
   CI_ELASTIC_AGENT_DOCKER_IMAGE: "704479110758.dkr.ecr.eu-west-2.amazonaws.com/elastic-agent"
 
 concurrency:
@@ -106,7 +106,8 @@ jobs:
 
   k8s_functional_tests:
     # Run only selected tests on PRs
-    if: github.event_name == 'pull_request'
+    # if: github.event_name == 'pull_request'
+    if: ${{ false }}
     name: ${{ matrix.test-target }}-${{ matrix.kind-config }}
     needs: [ Build ]
     runs-on: ubuntu-22.04