Merge branch '8.x' into mergify/bp/8.x/pr-5878

.buildkite/scripts/steps/beats_tests.sh

@@ -28,7 +28,7 @@ run_test_for_beat(){
     export WORKSPACE=$(pwd)
 
     set +e
-    AGENT_STACK_VERSION="8.16.0-SNAPSHOT" TEST_INTEG_CLEAN_ON_EXIT=true TEST_PLATFORMS="linux/amd64" STACK_PROVISIONER="$STACK_PROVISIONER" SNAPSHOT=true mage integration:testBeatServerless $beat_name
+    TEST_INTEG_CLEAN_ON_EXIT=true TEST_PLATFORMS="linux/amd64" STACK_PROVISIONER="$STACK_PROVISIONER" SNAPSHOT=true mage integration:testBeatServerless $beat_name
     TESTS_EXIT_STATUS=$?
     set -e
 

.buildkite/scripts/steps/integration-package.sh

@@ -3,4 +3,4 @@ set -euo pipefail
 
 source .buildkite/scripts/common.sh
 
-AGENT_PACKAGE_VERSION=8.16.0 PACKAGES=tar.gz,zip,rpm,deb PLATFORMS=linux/amd64,linux/arm64,windows/amd64  SNAPSHOT=true EXTERNAL=true  DEV=true mage package
+PACKAGES=tar.gz,zip,rpm,deb PLATFORMS=linux/amd64,linux/arm64,windows/amd64  SNAPSHOT=true EXTERNAL=true  DEV=true mage package

.buildkite/scripts/steps/integration_tests.sh

@@ -19,7 +19,7 @@ fi
 
 # Run integration tests
 set +e
-AGENT_VERSION="8.16.0-SNAPSHOT" AGENT_STACK_VERSION="8.16.0-SNAPSHOT" TEST_INTEG_CLEAN_ON_EXIT=true  STACK_PROVISIONER="$STACK_PROVISIONER" SNAPSHOT=true mage $MAGE_TARGET $MAGE_SUBTARGET
+AGENT_STACK_VERSION="${STACK_VERSION}" TEST_INTEG_CLEAN_ON_EXIT=true  STACK_PROVISIONER="$STACK_PROVISIONER" SNAPSHOT=true mage $MAGE_TARGET $MAGE_SUBTARGET
 TESTS_EXIT_STATUS=$?
 set -e
 

.buildkite/scripts/steps/k8s-extended-tests.sh

@@ -25,8 +25,8 @@ else
   exit 10
 fi
 
-AGENT_PACKAGE_VERSION="8.16.0" DEV=true SNAPSHOT=true EXTERNAL=true PACKAGES=docker mage -v package
-AGENT_VERSION="8.16.0-SNAPSHOT" AGENT_STACK_VERSION="8.16.0-SNAPSHOT" TEST_INTEG_CLEAN_ON_EXIT=true INSTANCE_PROVISIONER=kind STACK_PROVISIONER=stateful SNAPSHOT=true mage integration:kubernetesMatrix
+DEV=true SNAPSHOT=true EXTERNAL=true PACKAGES=docker mage -v package
+TEST_INTEG_CLEAN_ON_EXIT=true INSTANCE_PROVISIONER=kind STACK_PROVISIONER=stateful SNAPSHOT=true mage integration:kubernetesMatrix
 TESTS_EXIT_STATUS=$?
 set -e
 

deploy/helm/elastic-agent/README.md

@@ -123,12 +123,24 @@ The chart built-in [kubernetes integration](https://docs.elastic.co/integrations
 | kubernetes.system.metrics.enabled | bool | `true` | enable system metric stream (kubelet) [ref](https://www.elastic.co/docs/current/integrations/kubernetes/kubelet#system) |
 | kubernetes.system.metrics.vars | object | `{}` | system metric stream vars |
 
-### 3 - User Extra Integrations
+### 4 - System integration
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| system.enabled | bool | `false` | enable System integration. |
+| system.output | string | `"default"` | name of the output used in System integration. Note that this output needs to be defined in [outputs](#1-outputs) |
+| system.namespace | string | `"default"` | output namespace |
+| system.authLogs.enabled | bool | `true` | enable auth logs |
+| system.authLogs.vars | object | `{}` | override default variables of auth logs stream. Look in the [values.schema.json](values.schema.json) to see the available variables |
+| system.syslog.enabled | bool | `true` | enable syslog |
+| system.syslog.vars | object | `{}` | override default variables of syslog stream. Look in the [values.schema.json](values.schema.json) to see the available variables |
+| system.metrics.enabled | bool | `true` | enable metrics |
+
+### 5 - User Extra Integrations
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
 | extraIntegrations | object | `{}` | extra [user-defined integrations](https://www.elastic.co/guide/en/fleet/current/elastic-agent-input-configuration.html) to be added to the Elastic Agent An example can be found [here](./examples/nginx-custom-integration/README.md) |
 
-### 3 - Elastic-Agent Configuration
+### 6 - Elastic-Agent Configuration
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
 | agent.version | string | `"9.0.0"` | elastic-agent version |
@@ -138,7 +150,7 @@ The chart built-in [kubernetes integration](https://docs.elastic.co/integrations
 | agent.unprivileged | bool | `false` | enable unprivileged mode |
 | agent.presets | map[string]{} | `{ "perNode" : {...}, "clusterWide": {...}, "ksmSharded": {...} }` | Map of deployment presets for the Elastic Agent. The key of the map is the name of the preset. See more for the presets required by the built-in Kubernetes integration [here](./values.yaml) |
 
-### 3.1 - Elastic-Agent Managed Configuration
+### 6.1 - Elastic-Agent Managed Configuration
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
 | agent.fleet.enabled | bool | `false` | enable elastic-agent managed |

deploy/helm/elastic-agent/examples/README.md

@@ -3,6 +3,7 @@
 Here is a collection of example configurations for the eck-integrations chart.
 
 - [Kubernetes - defaults](kubernetes-default/README.md)
+- [System - custom authlogs path](kubernetes-default/README.md)
 - [Kubernetes - only container logs](kubernetes-only-logs/README.md)
 - [Kubernetes - only hints autodiscover](kubernetes-hints-autodiscover/README.md)
 - [Custom Integration - nginx](nginx-custom-integration/README.md)

deploy/helm/elastic-agent/examples/system-custom-auth-paths/README.md

@@ -0,0 +1,25 @@
+# Example: System Integration with custom auth log paths
+
+In this example we install the built-in `system` integration and specify custom paths for the auth logs stream (see [agent-system-values.yaml](agent-system-values.yaml)).
+
+## Prerequisites:
+1. A k8s secret that contains the connection details to an Elasticsearch cluster such as the URL and the API key ([Kibana - Creating API Keys](https://www.elastic.co/guide/en/kibana/current/api-keys.html)):
+    ```console
+    kubectl create secret generic es-api-secret \
+       --from-literal=api_key=... \
+       --from-literal=url=...
+    ```
+
+2. `system` integration assets installed through Kibana ([Kibana - Install and uninstall Elastic Agent integration assets](https://www.elastic.co/guide/en/fleet/current/install-uninstall-integration-assets.html))
+
+## Run:
+```console
+helm install elastic-agent ../../ \
+     -f ./agent-system-values.yaml \
+     --set outputs.default.type=ESSecretAuthAPI \
+     --set outputs.default.secretName=es-api-secret
+```
+
+## Validate:
+
+1. The Kibana `system`-related dashboards should start showing up the respective info.

deploy/helm/elastic-agent/examples/system-custom-auth-paths/agent-system-values.yaml

@@ -0,0 +1,13 @@
+system:
+  enabled: true
+  authLogs:
+    vars:
+      paths:
+        - /var/log/custom_auth.log
+  syslog:
+    vars:
+      paths:
+        - /var/log/custom_syslog.log
+
+agent:
+  unprivileged: true