Update event logger configuration via Fleet

.agent-versions.json

@@ -1,10 +1,10 @@
 {
   "testVersions": [
-    "8.14.0-SNAPSHOT",
-    "8.13.5-SNAPSHOT",
+    "8.15.0-SNAPSHOT",
+    "8.14.3-SNAPSHOT",
+    "8.14.2",
     "8.13.4",
-    "8.12.2",
-    "7.17.22-SNAPSHOT",
-    "7.17.21"
+    "7.17.23-SNAPSHOT",
+    "7.17.22"
   ]
 }

.buildkite/hooks/pre-command

@@ -46,7 +46,8 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-agent-package" ]]; then
   fi
 fi
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-agent-extended-testing" && "$BUILDKITE_STEP_KEY" == *"integration-tests"* ]]; then
+if [[ "$BUILDKITE_STEP_KEY" == *"integration-tests"* ]]; then
+  echo "Setting credentials"
   # Set GCP credentials
   export GOOGLE_APPLICATION_GCP_SECRET=$(retry 5 vault kv get -format=json -field=data ${CI_GCP_OBS_PATH})
   echo "${GOOGLE_APPLICATION_GCP_SECRET}" > ./gcp.json

.buildkite/hooks/pre-exit

@@ -2,7 +2,7 @@
 
 set -eo pipefail
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-agent-extended-testing" && "$BUILDKITE_STEP_KEY" == *"integration-tests"* ]]; then
+if [[ "$BUILDKITE_STEP_KEY" == *"integration-tests"* ]]; then
   if [[ -z "${WORKSPACE-""}" ]]; then
       WORKSPACE=$(git rev-parse --show-toplevel)
   fi

.buildkite/pipeline.integration-test-matrix.yml

@@ -0,0 +1,16 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
+
+env:
+  DOCKER_REGISTRY: "docker.elastic.co"
+  VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp"
+
+steps:
+  - label: "Integration test matrix"
+    key: "integration-tests-matrix"
+    command: ".buildkite/scripts/steps/integration_tests.sh stateful integration:matrix"
+    artifact_paths:
+      - "build/TEST-**"
+      - "build/diagnostics/*"
+    agents:
+      provider: "gcp"
+      machineType: "n1-standard-8"

.buildkite/pipeline.yml

@@ -191,10 +191,10 @@ steps:
         matrix:
           setup:
             k8s_version:
-              - "1.29.0"
-              - "1.28.0"
-              - "1.27.3"
-              - "1.26.6"
+              - "1.30.0"
+              - "1.29.4"
+              - "1.28.9"
+              - "1.27.13"
         retry:
           manual:
             allowed: true
@@ -232,3 +232,51 @@ steps:
                   image: "family/core-ubuntu-2204"
                 env:
                   - GH_VERSION=2.4.0
+
+  # Trigger for pull requests
+  - label: "Trigger Integration tests for Pull request"
+    if: |
+      (build.pull_request.id != null && !build.env("GITHUB_PR_LABELS") =~ /skip-it/) ||
+      build.env("GITHUB_PR_TRIGGER_COMMENT") =~ /.*extended.*/
+
+    plugins:
+      - monorepo-diff#v1.0.1:
+          diff: "git diff --name-only origin/${GITHUB_PR_TARGET_BRANCH}...HEAD"
+          watch:
+            - path:
+                - internal/
+                - dev-tools/
+                - pkg/
+                - testing/
+                - version/
+                - specs/
+                - .agent-versions.json
+                - .go-version
+                - .package-version
+                - go.mod
+                - go.sum
+                - magefile.go
+                - main.go
+
+                - .buildkite/integration.pipeline.yml
+                - .buildkite/pipeline.yml
+                - .buildkite/scripts/
+                - .buildkite/hooks/
+
+              config:
+                trigger: "elastic-agent-extended-testing"
+                build:
+                  commit: "${BUILDKITE_COMMIT}"
+                  branch: "${BUILDKITE_BRANCH}"
+                  env:
+                    - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST}
+                    - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
+                    - GITHUB_PR_LABELS=${GITHUB_PR_LABELS}
+
+  # Trigger for branches
+  - label: "Triggering Integration tests for branches"
+    if: build.pull_request.id == null
+    trigger: "elastic-agent-extended-testing"
+    build:
+      commit: "${BUILDKITE_COMMIT}"
+      branch: "${BUILDKITE_BRANCH}"

.buildkite/pull-requests.json

@@ -5,40 +5,23 @@
             "pipelineSlug": "elastic-agent",
             "allow_org_users": true,
             "allowed_repo_permissions": ["admin", "write"],
-            "allowed_list": ["dependabot[bot]", "mergify[bot]"],
+            "allowed_list": ["dependabot[bot]", "mergify[bot]", "github-actions[bot]", "elastic-vault-github-plugin-prod[bot]"],
             "set_commit_status": true,
             "build_on_commit": true,
             "build_on_comment": true,
-            "trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))|^/test$",
-            "always_trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))|^/test$",
+            "trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it|extended))|^/test\\W*(?:extended|)",
+            "always_trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it|extended))|^/test\\W*(?:extended|)",
             "skip_ci_labels": [ "skip-ci" ],
             "skip_target_branches": [ ],
-            "skip_ci_on_only_changed": [ "^.ci/", "^.github/", "^changelog", "^docs/", "\\.md$", "^docker-compose.yml", "^.pre-commit-config.yaml", "skaffold.yaml", "^Dockerfile.skaffold", "^Dockerfile"],
-            "always_require_ci_on_changed": [ ]
-        },
-        {
-            "enabled": true,
-            "build_drafts": false,
-            "pipelineSlug": "elastic-agent-extended-testing",
-            "allow_org_users": true,
-            "allowed_repo_permissions": ["admin", "write"],
-            "allowed_list": ["dependabot[bot]", "mergify[bot]"],
-            "set_commit_status": true,
-            "build_on_commit": true,
-            "build_on_comment": true,
-            "trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:extended))|^/test extended$",
-            "always_trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:extended))|^/test extended$",
-            "skip_ci_labels": [ "skip-ci", "skip-it" ],
-            "skip_target_branches": [ ],
-            "skip_ci_on_only_changed": [ "^.ci/", "^.github/", "^changelog", "^docs/", "\\.md$", "^sonar-project.properties", "^docker-compose.yml", "^.pre-commit-config.yaml", "skaffold.yaml", "^Dockerfile.skaffold", "^Dockerfile"],
+            "skip_ci_on_only_changed": [ "^.ci/", "^changelog", "^docs/", "\\.md$", "^docker-compose.yml", "^.pre-commit-config.yaml", "skaffold.yaml", "^Dockerfile.skaffold", "^Dockerfile"],
             "always_require_ci_on_changed": [ ]
         },
         {
             "enabled": true,
             "pipelineSlug": "elastic-agent-package",
             "allow_org_users": true,
             "allowed_repo_permissions": ["admin", "write"],
-            "allowed_list": [],
+            "allowed_list": ["dependabot[bot]", "mergify[bot]", "elastic-vault-github-plugin-prod[bot]"],
             "set_commit_status": false,
             "build_on_commit": false,
             "build_on_comment": true,
@@ -54,7 +37,7 @@
             "pipelineSlug": "elastic-agent-binary-dra",
             "allow_org_users": true,
             "allowed_repo_permissions": ["admin", "write"],
-            "allowed_list": [],
+            "allowed_list": ["dependabot[bot]", "mergify[bot]", "elastic-vault-github-plugin-prod[bot]"],
             "set_commit_status": false,
             "build_on_commit": false,
             "build_on_comment": true,

.buildkite/scripts/steps/sync-k8s.sh

@@ -26,4 +26,7 @@ make ci-clone-kibana-repository
 cp Makefile ./kibana
 cd kibana
 echo "--- Create Kibana PR"
-make ci-create-kubernetes-templates-pull-request
+make ci-create-kubernetes-templates-pull-request
+
+echo "--- [File Update] Kustomize-Tempates"
+GENERATEKUSTOMIZE=true make ci-create-kustomize 

.github/dependabot.yml

@@ -21,6 +21,7 @@ updates:
       - dependency-name: "github.com/elastic/*"
       - dependency-name: "go.opentelemetry.io/collector/*"
       - dependency-name: "github.com/open-telemetry/opentelemetry-collector-contrib/*"
+      - dependency-name: "github.com/elastic/opentelemetry-collector-components/*"
     reviewers:
       - "elastic/elastic-agent-control-plane"
     open-pull-requests-limit: 10

.github/workflows/bump-agent-versions.sh

@@ -16,6 +16,9 @@ else
         echo "Another PR for $GITHUB_REF_NAME is in review, skipping..."
         exit 0
     fi
+    # the mage target above requires to be on a release branch
+    # so, the new branch should not be created before the target is run
+    git checkout -b update-agent-versions-$GITHUB_RUN_ID
     git add .agent-versions.json .package-version
 
     nl=$'\n' # otherwise the new line character is not recognized properly

.github/workflows/bump-agent-versions.yml

@@ -26,8 +26,10 @@ jobs:
         with:
           go-version: 1.21
 
-      - name: Set up branch
-        run: git checkout -b update-agent-versions-$GITHUB_RUN_ID
+      - name: Set git config
+        run: |
+          git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git config --global user.name "github-actions[bot]"
 
       - name: Install mage
         uses: magefile/mage-action@v3
@@ -83,4 +85,3 @@ jobs:
         env:
           SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
           SLACK_MESSAGE: "Update for Elastic Agent versions has been created: ${{ steps.update.outputs.pr }}"
-

.github/workflows/bump-golang.yml

@@ -22,7 +22,7 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Install Updatecli in the runner
-        uses: updatecli/updatecli-action@fa41baa922561b436c449de1a0bd1f5bd768248c # v0.76.1
+        uses: updatecli/updatecli-action@eb158f6fd9e425b940a6750d6318f98e050ac390 # v0.76.1
 
       - name: Run Updatecli in Apply mode
         run: updatecli apply --config .github/updatecli-bump-golang.yml

.github/workflows/fragment-in-pr.yml

@@ -8,7 +8,7 @@ jobs:
     if: "!contains(github.event.pull_request.labels.*.name, 'skip-changelog') && !contains(github.event.pull_request.labels.*.name, 'backport')"
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - uses: actions/setup-go@v5
       with:
         go-version-file: .go-version

.github/workflows/golangci-lint.yml

@@ -18,7 +18,7 @@ jobs:
     name: lint
     runs-on: ${{  matrix.os }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - uses: actions/setup-go@v5
         with:

.github/workflows/post-dependabot.yml

@@ -16,7 +16,7 @@ jobs:
       contents: write
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - uses: actions/setup-go@v5
         with:

.go-version

@@ -1 +1 @@
-1.21.10
+1.22.5

.golangci.yml

@@ -116,7 +116,7 @@ linters-settings:
 
   gosimple:
     # Select the Go version to target. The default is '1.13'.
-    go: "1.21.10"
+    go: "1.22.5"
 
   nakedret:
     # make an issue if func has more lines of code than this setting and it has naked returns; default is 30
@@ -136,17 +136,17 @@ linters-settings:
 
   staticcheck:
     # Select the Go version to target. The default is '1.13'.
-    go: "1.21.10"
+    go: "1.22.5"
     checks: ["all"]
 
   stylecheck:
     # Select the Go version to target. The default is '1.13'.
-    go: "1.21.10"
+    go: "1.22.5"
     checks: ["all"]
 
   unused:
     # Select the Go version to target. The default is '1.13'.
-    go: "1.21.10"
+    go: "1.22.5"
 
   gosec:
     excludes:

.mergify.yml

@@ -319,3 +319,16 @@ pull_request_rules:
         labels:
           - "backport"
         title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}"
+  - name: backport patches to 8.15 branch
+    conditions:
+      - merged
+      - label=backport-v8.15.0
+    actions:
+      backport:
+        assignees:
+          - "{{ author }}"
+        branches:
+          - "8.15"
+        labels:
+          - "backport"
+        title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}"

Dockerfile

@@ -1,4 +1,4 @@
-ARG GO_VERSION=1.21.10
+ARG GO_VERSION=1.22.5
 FROM circleci/golang:${GO_VERSION}
 
 

Dockerfile.skaffold

@@ -1,4 +1,4 @@
-ARG GO_VERSION=1.21.10
+ARG GO_VERSION=1.22.5
 ARG crossbuild_image="docker.elastic.co/beats-dev/golang-crossbuild"
 ARG AGENT_VERSION=8.9.0-SNAPSHOT
 ARG AGENT_IMAGE="docker.elastic.co/beats/elastic-agent"

Makefile

@@ -36,6 +36,7 @@ notice:
 check-ci:
 	@mage -v check
 	@$(MAKE) notice
+	@GENERATEKUSTOMIZE=true $(MAKE) -C deploy/kubernetes generate-k8s
 	@$(MAKE) -C deploy/kubernetes generate-k8s
 	@$(MAKE) check-no-changes