diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index c23f5c3af645..7be1f965230c 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -34,7 +34,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == fi fi -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-winlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" ]]; then +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-winlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-winlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-dockerlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-auditbeat" ]]; then source .buildkite/scripts/setenv.sh if [[ "${BUILDKITE_COMMAND}" =~ ^buildkite-agent ]]; then echo "Skipped pre-command when running the Upload pipeline" diff --git a/.buildkite/scripts/cloud_tests.sh b/.buildkite/scripts/cloud_tests.sh index 9e7ce95be661..3baeb1be1a35 100755 --- a/.buildkite/scripts/cloud_tests.sh +++ b/.buildkite/scripts/cloud_tests.sh @@ -9,9 +9,6 @@ set -euo pipefail trap 'teardown || true; unset_secrets' EXIT -# Set the MODULE env variable if possible -defineModuleFromTheChangeSet "${BEATS_PROJECT_NAME}" - # Prepare the cloud resources using Terraform startCloudTestEnv "${MODULE_DIR}" diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index d6a91a482436..b6170070836f 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -11,20 +11,34 @@ GITHUB_PR_LABELS=${GITHUB_PR_LABELS:-""} ONLY_DOCS=${ONLY_DOCS:-"true"} OSS_MODULE_PATTERN="^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" +# define if needed run the whole pipeline for the particular beat [ -z "${run_libbeat+x}" ] && run_libbeat="$(buildkite-agent meta-data get run_libbeat --default "false")" [ -z "${run_metricbeat+x}" ] && run_metricbeat="$(buildkite-agent meta-data get run_metricbeat --default "false")" [ -z "${run_packetbeat+x}" ] && run_packetbeat="$(buildkite-agent meta-data get run_packetbeat --default "false")" [ -z "${run_winlogbeat+x}" ] && run_winlogbeat="$(buildkite-agent meta-data get run_winlogbeat --default "false")" +[ -z "${run_xpack_libbeat+x}" ] && run_xpack_libbeat="$(buildkite-agent meta-data get run_xpack_libbeat --default "false")" +[ -z "${run_xpack_metricbeat+x}" ] && run_xpack_metricbeat="$(buildkite-agent meta-data get run_xpack_metricbeat --default "false")" +[ -z "${run_xpack_packetbeat+x}" ] && run_xpack_packetbeat="$(buildkite-agent meta-data get run_xpack_packetbeat --default "false")" +[ -z "${run_xpack_winlogbeat+x}" ] && run_xpack_winlogbeat="$(buildkite-agent meta-data get run_xpack_winlogbeat --default "false")" +[ -z "${run_xpack_auditbeat+x}" ] && run_xpack_auditbeat="$(buildkite-agent meta-data get run_xpack_auditbeat --default "false")" + +# define if needed run ARM platform-specific tests for the particular beat [ -z "${run_libbeat_arm_tests+x}" ] && run_libbeat_arm_tests="$(buildkite-agent meta-data get run_libbeat_arm_tests --default "false")" [ -z "${run_packetbeat_arm_tests+x}" ] && run_packetbeat_arm_tests="$(buildkite-agent meta-data get run_packetbeat_arm_tests --default "false")" +[ -z "${run_xpack_auditbeat_arm_tests+x}" ] && run_xpack_auditbeat_arm_tests="$(buildkite-agent meta-data get run_xpack_auditbeat_arm_tests --default "false")" +[ -z "${run_xpack_libbeat_arm_tests+x}" ] && run_xpack_libbeat_arm_tests="$(buildkite-agent meta-data get run_xpack_libbeat_arm_tests --default "false")" +[ -z "${run_xpack_packetbeat_arm_tests+x}" ] && run_xpack_packetbeat_arm_tests="$(buildkite-agent meta-data get run_xpack_packetbeat_arm_tests --default "false")" + +# define if needed run MacOS platform-specific tests for the particular beat [ -z "${run_metricbeat_macos_tests+x}" ] && run_metricbeat_macos_tests="$(buildkite-agent meta-data get run_metricbeat_macos_tests --default "false")" [ -z "${run_packetbeat_macos_tests+x}" ] && run_packetbeat_macos_tests="$(buildkite-agent meta-data get run_packetbeat_macos_tests --default "false")" -[ -z "${run_xpack_libbeat_arm_tests+x}" ] && run_xpack_libbeat_arm_tests="$(buildkite-agent meta-data get run_xpack_libbeat_arm_tests --default "false")" -[ -z "${run_xpack_metricbeat_aws_tests+x}" ] && run_xpack_metricbeat_aws_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_aws_tests --default "false")" +[ -z "${run_xpack_auditbeat_macos_tests+x}" ] && run_xpack_auditbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_auditbeat_macos_tests --default "false")" [ -z "${run_xpack_metricbeat_macos_tests+x}" ] && run_xpack_metricbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_macos_tests --default "false")" -[ -z "${run_xpack_packetbeat_arm_tests+x}" ] && run_xpack_packetbeat_arm_tests="$(buildkite-agent meta-data get run_xpack_packetbeat_arm_tests --default "false")" [ -z "${run_xpack_packetbeat_macos_tests+x}" ] && run_xpack_packetbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_packetbeat_macos_tests --default "false")" +# define if needed run cloud-specific tests for the particular beat +[ -z "${run_xpack_metricbeat_aws_tests+x}" ] && run_xpack_metricbeat_aws_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_aws_tests --default "false")" + metricbeat_changeset=( "^metricbeat/.*" ) @@ -41,6 +55,14 @@ winlogbeat_changeset=( "^winlogbeat/.*" ) +xpack_auditbeat_changeset=( + "^x-pack/auditbeat/.*" + ) + +xpack_dockerlogbeat_changeset=( + "^x-pack/dockerlogbeat/.*" + ) + xpack_libbeat_changeset=( "^x-pack/libbeat/.*" ) @@ -101,6 +123,12 @@ case "${BUILDKITE_PIPELINE_SLUG}" in "beats-winlogbeat") BEAT_CHANGESET_REFERENCE=${winlogbeat_changeset[@]} ;; + "beats-xpack-auditbeat") + BEAT_CHANGESET_REFERENCE=${xpack_auditbeat_changeset[@]} + ;; + "beats-xpack-dockerlogbeat") + BEAT_CHANGESET_REFERENCE=${xpack_dockerlogbeat_changeset[@]} + ;; "beats-xpack-libbeat") BEAT_CHANGESET_REFERENCE=${xpack_libbeat_changeset[@]} ;; @@ -110,6 +138,9 @@ case "${BUILDKITE_PIPELINE_SLUG}" in "beats-xpack-packetbeat") BEAT_CHANGESET_REFERENCE=${xpack_packetbeat_changeset[@]} ;; + "beats-xpack-winlogbeat") + BEAT_CHANGESET_REFERENCE=${xpack_winlogbeat_changeset[@]} + ;; *) echo "The changeset for the ${BUILDKITE_PIPELINE_SLUG} pipeline hasn't been defined yet." ;; @@ -273,10 +304,10 @@ with_dependencies() { if [ "${platform_type}" == "Linux" ]; then if [ "${linuxType}" = "ubuntu" ]; then sudo apt-get update - sudo apt-get install -y libsystemd-dev libpcap-dev + sudo apt-get install -y libsystemd-dev libpcap-dev librpm-dev elif [ "${linuxType}" = "rhel" ]; then # sudo dnf update -y - sudo dnf install -y systemd-devel + sudo dnf install -y systemd-devel rpm-devel wget https://mirror.stream.centos.org/9-stream/CRB/${arch_type}/os/Packages/libpcap-devel-1.10.0-4.el9.${arch_type}.rpm #TODO: move this step to our own image sudo dnf install -y libpcap-devel-1.10.0-4.el9.${arch_type}.rpm #TODO: move this step to our own image fi @@ -357,7 +388,7 @@ are_conditions_met_mandatory_tests() { are_conditions_met_arm_tests() { if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 - if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then + if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-auditbeat" ]]; then if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_ARM_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_ARM_LABEL} || "${!TRIGGER_SPECIFIC_ARM_TESTS}" == "true" ]]; then return 0 fi @@ -368,7 +399,7 @@ are_conditions_met_arm_tests() { are_conditions_met_macos_tests() { if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 - if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then + if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-auditbeat" ]]; then if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_MACOS_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_MACOS_LABEL} || "${!TRIGGER_SPECIFIC_MACOS_TESTS}" == "true" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 return 0 fi @@ -501,4 +532,9 @@ if are_paths_changed "${packaging_changeset[@]}" ; then export PACKAGING_CHANGES="true" fi +if [[ "$BUILDKITE_STEP_KEY" == "xpack-winlogbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-metricbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-dockerlogbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "metricbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-auditbeat-pipeline" ]]; then + # Set the MODULE env variable if possible, it should be defined before generating pipeline's steps. It is used in multiple pipelines. + defineModuleFromTheChangeSet "${BEATS_PROJECT_NAME}" +fi + check_and_set_beat_vars diff --git a/.buildkite/scripts/generate_xpack_auditbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_auditbeat_pipeline.sh new file mode 100755 index 000000000000..f29e6152e602 --- /dev/null +++ b/.buildkite/scripts/generate_xpack_auditbeat_pipeline.sh @@ -0,0 +1,166 @@ +#!/usr/bin/env bash + +source .buildkite/scripts/common.sh + +set -euo pipefail + +pipelineName="pipeline.xpack-auditbeat-dynamic.yml" + +echo "Add the mandatory and extended tests without additional conditions into the pipeline" +if are_conditions_met_mandatory_tests; then + cat > $pipelineName <<- YAML + +steps: + + - group: "Mandatory Tests" + key: "mandatory-tests" + steps: + + - label: ":linux: Ubuntu Unit (MODULE) Tests" + key: "mandatory-linux-unit-test" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" + env: + MODULE: $MODULE + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + + - label: ":rhel: RHEL-9 Unit Tests" + key: "mandatory-rhel9-unit-test" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" + agents: + provider: "gcp" + image: "${IMAGE_RHEL9_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + + - label: ":windows: Windows Unit Tests - {{matrix.image}}" + command: ".buildkite/scripts/win_unit_tests.ps1" + key: "mandatory-win-unit-tests" + agents: + provider: "gcp" + image: "{{matrix.image}}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + matrix: + setup: + image: + - "${IMAGE_WIN_2016}" + - "${IMAGE_WIN_2022}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +## TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h + + - group: "Extended Windows Tests" + key: "extended-win-tests" + steps: + + - label: ":windows: Windows Unit Tests - {{matrix.image}}" + command: ".buildkite/scripts/win_unit_tests.ps1" + key: "extended-win-unit-tests" + agents: + provider: "gcp" + image: "{{matrix.image}}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + matrix: + setup: + image: + - "${IMAGE_WIN_10}" + - "${IMAGE_WIN_11}" + - "${IMAGE_WIN_2019}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + +YAML +else + echo "The conditions don't match to requirements for generating pipeline steps." + exit 0 +fi + +if are_conditions_met_arm_tests || are_conditions_met_macos_tests ; then + cat >> $pipelineName <<- YAML + + - group: "Extended Tests" + key: "extended-tests" + steps: + +YAML +fi + +if are_conditions_met_macos_tests; then + cat >> $pipelineName <<- YAML + + - label: ":mac: MacOS Unit Tests" + key: "extended-macos-unit-tests" + command: ".buildkite/scripts/unit_tests.sh" + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_X86_64}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +YAML +fi + +if are_conditions_met_arm_tests; then + cat >> $pipelineName <<- YAML + - label: ":linux: ARM Ubuntu Unit Tests" + key: "extended-arm64-unit-test" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +YAML +fi + +echo "Check and add the Packaging into the pipeline" +if are_conditions_met_packaging; then + cat >> $pipelineName <<- YAML + + - wait: ~ + depends_on: + - step: "mandatory-tests" + allow_failure: false + + - group: "Packaging" # TODO: check conditions for future the main pipeline migration: https://github.com/elastic/beats/pull/28589 + key: "packaging" + steps: + - label: ":linux: Packaging Linux" + key: "packaging-linux" + command: "cd $BEATS_PROJECT_NAME && mage package" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + env: + PLATFORMS: "${PACKAGING_PLATFORMS}" + + - label: ":linux: Packaging ARM" + key: "packaging-arm" + command: "cd $BEATS_PROJECT_NAME && mage package" + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + env: + PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" + PACKAGES: "docker" + +YAML +fi + +echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public +cat $pipelineName + +echo "--- Loading dynamic steps" +buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/generate_xpack_dockerlogbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_dockerlogbeat_pipeline.sh new file mode 100755 index 000000000000..46e92f8ddf97 --- /dev/null +++ b/.buildkite/scripts/generate_xpack_dockerlogbeat_pipeline.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash + +source .buildkite/scripts/common.sh + +set -euo pipefail + +pipelineName="pipeline.xpack-dockerlogbeat-dynamic.yml" + +echo "Add the mandatory and extended tests without additional conditions into the pipeline" +if are_conditions_met_mandatory_tests; then + cat > $pipelineName <<- YAML + +steps: + + - group: "Mandatory Tests" + key: "mandatory-tests" + steps: + - label: ":linux: Ubuntu Unit Tests" + key: "mandatory-linux-unit-test" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + + - label: ":go: Go Integration Tests" + key: "mandatory-int-test" + command: "cd $BEATS_PROJECT_NAME && mage goIntegTest" + env: + MODULE: $MODULE + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + +YAML +fi + +echo "Check and add the Packaging into the pipeline" +if are_conditions_met_packaging; then + cat >> $pipelineName <<- YAML + + - wait: ~ + depends_on: + - step: "mandatory-tests" + allow_failure: false + + - group: "Packaging" # TODO: check conditions for future the main pipeline migration: https://github.com/elastic/beats/pull/28589 + key: "packaging" + steps: + - label: ":linux: Packaging Linux" + key: "packaging-linux" + command: "cd $BEATS_PROJECT_NAME && mage package" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + env: + PLATFORMS: "${PACKAGING_PLATFORMS}" + + - label: ":linux: Packaging ARM" + key: "packaging-arm" + command: "cd $BEATS_PROJECT_NAME && mage package" + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + env: + PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" + PACKAGES: "docker" + +YAML +fi + +echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public +cat $pipelineName + +echo "--- Loading dynamic steps" +buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index af116b17209d..ddc3ce2c8f20 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -27,6 +27,8 @@ steps: - label: ":go: Go Integration Tests" key: "mandatory-int-test" command: ".buildkite/scripts/go_int_tests.sh" + env: + MODULE: $MODULE agents: provider: "gcp" image: "${DEFAULT_UBUNTU_X86_64_IMAGE}" @@ -36,6 +38,8 @@ steps: - label: ":python: Python Integration Tests" key: "mandatory-python-int-test" command: ".buildkite/scripts/py_int_tests.sh" + env: + MODULE: $MODULE agents: provider: "gcp" image: "${DEFAULT_UBUNTU_X86_64_IMAGE}" @@ -131,6 +135,8 @@ if are_conditions_met_aws_tests; then - label: ":linux: Cloud Tests" key: "extended-cloud-test" command: ".buildkite/scripts/cloud_tests.sh" + env: + MODULE: $MODULE agents: provider: "gcp" image: "${DEFAULT_UBUNTU_X86_64_IMAGE}" diff --git a/.buildkite/scripts/generate_xpack_winlogbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_winlogbeat_pipeline.sh new file mode 100755 index 000000000000..108a70c15622 --- /dev/null +++ b/.buildkite/scripts/generate_xpack_winlogbeat_pipeline.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash + +source .buildkite/scripts/common.sh + +set -euo pipefail + +pipelineName="pipeline.xpack-winlogbeat-dynamic.yml" + +echo "Add the mandatory and extended tests without additional conditions into the pipeline" +if are_conditions_met_mandatory_tests; then + cat > $pipelineName <<- YAML + +steps: + + - group: "Mandatory Tests" + key: "mandatory-tests" + steps: + + - label: ":windows: Windows 2019 Unit (MODULE) Tests" + key: "mandatory-win-2019-unit-tests" + command: ".buildkite/scripts/win_unit_tests.ps1" + env: + MODULE: $MODULE + agents: + provider: "gcp" + image: "${IMAGE_WIN_2019}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":windows: Windows 2016/2022 Unit Tests - {{matrix.image}}" + command: ".buildkite/scripts/win_unit_tests.ps1" + key: "mandatory-win-unit-tests" + agents: + provider: "gcp" + image: "{{matrix.image}}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + matrix: + setup: + image: + - "${IMAGE_WIN_2016}" + - "${IMAGE_WIN_2022}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +# echo "Add the extended windows tests into the pipeline" +# TODO: ADD conditions from the main pipeline + + - group: "Extended Windows Tests" + key: "extended-win-tests" + steps: + + - label: ":windows: Windows Unit Tests - {{matrix.image}}" + command: ".buildkite/scripts/win_unit_tests.ps1" + key: "extended-win-unit-tests" + agents: + provider: "gcp" + image: "{{matrix.image}}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + matrix: + setup: + image: + - "${IMAGE_WIN_10}" + - "${IMAGE_WIN_11}" + - "${IMAGE_WIN_2019}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +YAML +else + echo "The conditions don't match to requirements for generating pipeline steps." + exit 0 +fi + +echo "Check and add the Packaging into the pipeline" +if are_conditions_met_packaging; then + cat >> $pipelineName <<- YAML + + - wait: ~ + depends_on: + - step: "mandatory-tests" + allow_failure: false + + - group: "Packaging" # TODO: check conditions for future the main pipeline migration: https://github.com/elastic/beats/pull/28589 + key: "packaging" + steps: + + - label: ":linux: Packaging Linux" + key: "packaging-linux" + command: "cd $BEATS_PROJECT_NAME && mage package" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + env: + PLATFORMS: "${PACKAGING_PLATFORMS}" + +YAML +fi + +echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public +cat $pipelineName + +echo "--- Loading dynamic steps" +buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh index 29a8a05446ec..e40f858f1c44 100755 --- a/.buildkite/scripts/setenv.sh +++ b/.buildkite/scripts/setenv.sh @@ -56,10 +56,16 @@ exportVars() { fi } -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-winlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-auditbeat" ]]; then exportVars export RACE_DETECTOR="true" export TEST_COVERAGE="true" export DOCKER_PULL="0" export TEST_TAGS="${TEST_TAGS:+$TEST_TAGS,}oracle" fi + +if [[ "$BUILDKITE_STEP_KEY" == "xpack-winlogbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-metricbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-dockerlogbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "metricbeat-pipeline" ]]; then + # Set the MODULE env variable if possible, it should be defined before generating pipeline's steps. It is used in multiple pipelines. + source .buildkite/scripts/common.sh + defineModuleFromTheChangeSet "${BEATS_PROJECT_NAME}" +fi diff --git a/.buildkite/x-pack/pipeline.xpack.auditbeat.yml b/.buildkite/x-pack/pipeline.xpack.auditbeat.yml index 2343eb6a4ddd..d88bf2a4ff05 100644 --- a/.buildkite/x-pack/pipeline.xpack.auditbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.auditbeat.yml @@ -1,6 +1,62 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json name: "beats-xpack-auditbeat" +env: + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" + DEFAULT_UBUNTU_X86_64_IMAGE: "family/core-ubuntu-2204" + IMAGE_RHEL9_X86_64: "family/platform-ingest-beats-rhel-9" + IMAGE_WIN_10: "family/general-windows-10" + IMAGE_WIN_11: "family/general-windows-11" + IMAGE_WIN_2016: "family/core-windows-2016" + IMAGE_WIN_2019: "family/core-windows-2019" + IMAGE_WIN_2022: "family/core-windows-2022" + IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" + GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" + GCP_WIN_MACHINE_TYPE: "n2-standard-8" + AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + BEATS_PROJECT_NAME: "x-pack/auditbeat" + steps: - - label: "Example test" - command: echo "Hello!" + + - input: "Input Parameters" + key: "force-run-stages" + fields: + - select: "Auditbeat - run_xpack_auditbeat" + key: "run_xpack_auditbeat" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "false" + - select: "Auditbeat - run_xpack_auditbeat_macos_tests" + key: "run_xpack_auditbeat_macos_tests" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "false" + - select: "Auditbeat - run_xpack_auditbeat_arm_tests" + key: "run_xpack_auditbeat_arm_tests" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "false" + + if: "build.source == 'ui'" + + - wait: ~ + if: "build.source == 'ui'" + allow_dependency_failure: false + + - label: ":linux: Load dynamic x-pack auditbeat pipeline" + key: "xpack-auditbeat-pipeline" + command: ".buildkite/scripts/generate_xpack_auditbeat_pipeline.sh" + notify: + - github_commit_status: + context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml b/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml index c4a0805615b1..bcc2610e175b 100644 --- a/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml @@ -1,6 +1,38 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json name: "beats-xpack-dockerlogbeat" +env: + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + DEFAULT_UBUNTU_X86_64_IMAGE: "family/core-ubuntu-2204" + IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" + GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" + GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" + AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + BEATS_PROJECT_NAME: "x-pack/dockerlogbeat" + steps: - - label: "Example test" - command: echo "Hello!" + + - input: "Input Parameters" + key: "force-run-stages" + fields: + - select: "Dockerlogbeat - run_xpack_dockerlogbeat" + key: "run_xpack_dockerlogbeat" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "false" + + if: "build.source == 'ui'" + + - wait: ~ + if: "build.source == 'ui'" + allow_dependency_failure: false + + - label: ":linux: Load dynamic x-pack dockerlogbeat pipeline" + key: "xpack-dockerlogbeat-pipeline" + command: ".buildkite/scripts/generate_xpack_dockerlogbeat_pipeline.sh" + notify: + - github_commit_status: + context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml index 81756564919d..216f31343446 100644 --- a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml @@ -54,7 +54,7 @@ steps: allow_dependency_failure: false - label: ":linux: Load dynamic x-pack metricbeat pipeline" - key: "metricbeat-pipeline" + key: "xpack-metricbeat-pipeline" command: ".buildkite/scripts/generate_xpack_metricbeat_pipeline.sh" notify: - github_commit_status: diff --git a/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml b/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml index 34321b61161b..5c8acefd6989 100644 --- a/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml @@ -1,5 +1,39 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json +name: "beats-xpack-winlogbeat" + +env: + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + IMAGE_WIN_10: "family/general-windows-10" + IMAGE_WIN_11: "family/general-windows-11" + IMAGE_WIN_2016: "family/core-windows-2016" + IMAGE_WIN_2019: "family/core-windows-2019" + IMAGE_WIN_2022: "family/core-windows-2022" + GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" + GCP_WIN_MACHINE_TYPE: "n2-standard-8" + BEATS_PROJECT_NAME: "x-pack/winlogbeat" steps: - - label: "Example test" - command: echo "Hello!" + + - input: "Input Parameters" + key: "force-run-stages" + fields: + - select: "Winlogbeat - run_xpack_winlogbeat" + key: "run_xpack_winlogbeat" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "false" + if: "build.source == 'ui'" + + - wait: ~ + if: "build.source == 'ui'" + allow_dependency_failure: false + + - label: ":linux: Load dynamic x-pack winlogbeat pipeline" + key: "xpack-winlogbeat-pipeline" + command: ".buildkite/scripts/generate_xpack_winlogbeat_pipeline.sh" + notify: + - github_commit_status: + context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index be217566c4c5..a1b1f8e84eb9 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -101,6 +101,8 @@ fields added to events containing the Beats version. {pull}37553[37553] - Fix "failed processing S3 event for object key" error on aws-s3 input when key contains the "+" character {issue}38012[38012] {pull}38125[38125] - Fix duplicated addition of regexp extension in CEL input. {pull}38181[38181] - Fix the incorrect values generated by the uri_parts processor. {pull}38216[38216] +- Fix HTTPJSON handling of empty object bodies in POST requests. {issue}33961[33961] {pull}38290[38290] +- Fix PEM key validation for CEL and HTTPJSON inputs. {pull}38405[38405] *Heartbeat* @@ -146,6 +148,7 @@ Setting environmental variable ELASTIC_NETINFO:false in Elastic Agent pod will d - Upgrade go-sysinfo from 1.12.0 to 1.13.1. {pull}37996[37996] - Make `range` condition work with numeric values as strings. {pull}38080[38080] - Allow users to configure number of output workers (for outputs that support workers) with either `worker` or `workers`. {pull}38257[38257] +- Kafka output now validates the `topics` and `topic` configuration values {pull}38058[38058] *Auditbeat* @@ -154,6 +157,7 @@ Setting environmental variable ELASTIC_NETINFO:false in Elastic Agent pod will d *Filebeat* +- Adding Saved Object name field to Kibana audit logs {pull}38307[38307] - Update SQL input documentation regarding Oracle DSNs {pull}37590[37590] - add documentation for decode_xml_wineventlog processor field mappings. {pull}32456[32456] - httpjson input: Add request tracing logger. {issue}32402[32402] {pull}32412[32412] @@ -230,6 +234,7 @@ Setting environmental variable ELASTIC_NETINFO:false in Elastic Agent pod will d - Add a `/inputs/` route to the HTTP monitoring endpoint that exposes metrics for each metricset instance. {pull}36971[36971] - Add linux IO metrics to system/process {pull}37213[37213] - Add new memory/cgroup metrics to Kibana module {pull}37232[37232] +- Add SSL support to mysql module {pull}37997[37997] *Metricbeat* @@ -268,6 +273,7 @@ Setting environmental variable ELASTIC_NETINFO:false in Elastic Agent pod will d *Filebeat* +- Deprecate `syslog` input in favor of `syslog` processor. {issue}37555[37555] {pull}38277[38277] *Heartbeat* diff --git a/NOTICE.txt b/NOTICE.txt index 300952e885e5..a838bbeaed9d 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -15223,11 +15223,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/go-sysinfo@v1.1 -------------------------------------------------------------------------------- Dependency : github.com/elastic/go-ucfg -Version: v0.8.6 +Version: v0.8.7 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/go-ucfg@v0.8.6/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/go-ucfg@v0.8.7/LICENSE: Apache License Version 2.0, January 2004 diff --git a/dev-tools/mage/crossbuild.go b/dev-tools/mage/crossbuild.go index f500611e5cc9..94b972ea25e0 100644 --- a/dev-tools/mage/crossbuild.go +++ b/dev-tools/mage/crossbuild.go @@ -327,7 +327,7 @@ func (b GolangCrossBuilder) Build() error { args = append(args, "--rm", - "--env", "GOFLAGS=-mod=readonly", + "--env", "GOFLAGS=-mod=readonly -buildvcs=false", "--env", "MAGEFILE_VERBOSE="+verbose, "--env", "MAGEFILE_TIMEOUT="+EnvOr("MAGEFILE_TIMEOUT", ""), "--env", fmt.Sprintf("SNAPSHOT=%v", Snapshot), diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index ddc887d246f2..4cf4b99b5e74 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -86793,6 +86793,17 @@ example: 6295bdd0-0a0e-11e7-825f-6748cda7d858 -- +*`kibana.saved_object.name`*:: ++ +-- +The name of the saved object associated with this event. + +type: keyword + +example: my-saved-object + +-- + *`kibana.add_to_spaces`*:: + -- diff --git a/filebeat/docs/inputs/input-syslog.asciidoc b/filebeat/docs/inputs/input-syslog.asciidoc index e43eabea378a..3e0555d03b91 100644 --- a/filebeat/docs/inputs/input-syslog.asciidoc +++ b/filebeat/docs/inputs/input-syslog.asciidoc @@ -3,6 +3,10 @@ [id="{beatname_lc}-input-{type}"] === Syslog input +deprecated:[8.14.0] + +The syslog input is deprecated. Please use the <> processor for processing syslog messages. + ++++ Syslog ++++ diff --git a/filebeat/input/syslog/input.go b/filebeat/input/syslog/input.go index 702472794dd6..c91158c0f894 100644 --- a/filebeat/input/syslog/input.go +++ b/filebeat/input/syslog/input.go @@ -28,6 +28,7 @@ import ( "github.com/elastic/beats/v7/filebeat/input" "github.com/elastic/beats/v7/filebeat/inputsource" "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/common/cfgwarn" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" @@ -85,6 +86,8 @@ var ( "local6", "local7", } + + deprecatedNotificationOnce sync.Once ) func init() { @@ -112,6 +115,10 @@ func NewInput( ) (input.Input, error) { log := logp.NewLogger("syslog") + deprecatedNotificationOnce.Do(func() { + cfgwarn.Deprecate("", "Syslog input. Use Syslog processor instead.") + }) + out, err := outlet.Connect(cfg) if err != nil { return nil, err @@ -180,7 +187,7 @@ func GetCbByConfig(cfg config, forwarder *harvester.Forwarder, log *logp.Logger) case syslogFormatRFC5424: return func(data []byte, metadata inputsource.NetworkMetadata) { ev := parseAndCreateEvent5424(data, metadata, cfg.Timezone.Location(), log) - forwarder.Send(ev) + _ = forwarder.Send(ev) } case syslogFormatAuto: @@ -191,7 +198,7 @@ func GetCbByConfig(cfg config, forwarder *harvester.Forwarder, log *logp.Logger) } else { ev = parseAndCreateEvent3164(data, metadata, cfg.Timezone.Location(), log) } - forwarder.Send(ev) + _ = forwarder.Send(ev) } case syslogFormatRFC3164: break @@ -199,7 +206,7 @@ func GetCbByConfig(cfg config, forwarder *harvester.Forwarder, log *logp.Logger) return func(data []byte, metadata inputsource.NetworkMetadata) { ev := parseAndCreateEvent3164(data, metadata, cfg.Timezone.Location(), log) - forwarder.Send(ev) + _ = forwarder.Send(ev) } } diff --git a/filebeat/module/kibana/_meta/fields.yml b/filebeat/module/kibana/_meta/fields.yml index d4e664ade58d..aed9252122c8 100644 --- a/filebeat/module/kibana/_meta/fields.yml +++ b/filebeat/module/kibana/_meta/fields.yml @@ -27,6 +27,10 @@ description: "The id of the saved object associated with this event." example: "6295bdd0-0a0e-11e7-825f-6748cda7d858" type: keyword + - name: saved_object.name + description: "The name of the saved object associated with this event." + example: "my-saved-object" + type: keyword - name: add_to_spaces description: "The set of space ids that a saved object was shared to." example: "['default', 'marketing']" diff --git a/filebeat/module/kibana/audit/test/test-audit-814.log b/filebeat/module/kibana/audit/test/test-audit-814.log new file mode 100644 index 000000000000..97127ddcbf0e --- /dev/null +++ b/filebeat/module/kibana/audit/test/test-audit-814.log @@ -0,0 +1,5 @@ +{"event":{"action":"saved_object_create","category":["database"],"outcome":"unknown","type":["access"]},"kibana":{"saved_object":{"id":"fleet-default-settings","type":"ingest_manager_settings"}},"labels":{"application":"elastic/fleet"},"service":{"node":{"roles":["background_tasks","ui"]}},"ecs":{"version":"8.6.1"},"@timestamp":"2023-06-19T15:18:47.298+00:00","message":"User is accessing ingest_manager_settings [id=fleet-default-settings]","log":{"level":"INFO","logger":"plugins.security.audit.ecs"},"process":{"pid":7},"trace":{"id":"809d3449277aba205a3ac539d23dbf7e"},"transaction":{"id":"49a38064b0f1dc1e"}} +{"event":{"action":"saved_object_create","category":["database"],"outcome":"unknown","type":["access"]},"kibana":{"saved_object":{"id":"a09a5397-7b9a-5a73-a622-e29f4c635658","type":"ingest-outputs"}},"labels":{"application":"elastic/fleet"},"service":{"node":{"roles":["background_tasks","ui"]}},"ecs":{"version":"8.6.1"},"@timestamp":"2023-06-19T15:18:48.987+00:00","message":"User is accessing ingest-outputs [id=a09a5397-7b9a-5a73-a622-e29f4c635658]","log":{"level":"INFO","logger":"plugins.security.audit.ecs"},"process":{"pid":7},"trace":{"id":"809d3449277aba205a3ac539d23dbf7e"},"transaction":{"id":"49a38064b0f1dc1e"}} +{"event":{"action":"saved_object_create","category":["database"],"outcome":"unknown","type":["access"]},"kibana":{"saved_object":{"id":"synthetics","type":"epm-packages"}},"labels":{"application":"elastic/fleet"},"service":{"node":{"roles":["background_tasks","ui"]}},"ecs":{"version":"8.6.1"},"@timestamp":"2023-06-19T15:18:53.426+00:00","message":"User is accessing epm-packages [id=synthetics]","log":{"level":"INFO","logger":"plugins.security.audit.ecs"},"process":{"pid":7},"trace":{"id":"809d3449277aba205a3ac539d23dbf7e"},"transaction":{"id":"49a38064b0f1dc1e"}} +{"event":{"action":"http_request","category":["web"],"outcome":"unknown"},"http":{"request":{"method":"get"}},"url":{"domain":"kibana","path":"/api/features","port":5601,"scheme":"http"},"user":{"name":"elastic","roles":["superuser"]},"kibana":{"space_id":"default"},"trace":{"id":"e2792f3f-4cf1-4f6d-b4eb-5b491724c295"},"client":{"ip":"172.22.0.2"},"service":{"node":{"roles":["background_tasks","ui"]}},"ecs":{"version":"8.6.1"},"@timestamp":"2023-06-19T15:19:18.882+00:00","message":"User is requesting [/api/features] endpoint","log":{"level":"INFO","logger":"plugins.security.audit.ecs"},"process":{"pid":7},"transaction":{"id":"cf44f52888b9ec5a"}} +{"event":{"action":"saved_object_create","category":["database"],"outcome":"unknown","type":["access"]},"kibana":{"saved_object":{"id":"abcde-fghijk","type":"ingest_manager_settings","name":"fleet-object-name"}},"labels":{"application":"elastic/fleet"},"service":{"node":{"roles":["background_tasks","ui"]}},"ecs":{"version":"8.6.1"},"@timestamp":"2023-06-19T16:18:47.298+00:00","message":"User is accessing ingest_manager_settings [id=fleet-default-settings]","log":{"level":"INFO","logger":"plugins.security.audit.ecs"},"process":{"pid":7},"trace":{"id":"809d3449277aba205a3ac539d23dbf7e"},"transaction":{"id":"49a38064b0f1dc1e"}} diff --git a/filebeat/module/kibana/audit/test/test-audit-814.log-expected.json b/filebeat/module/kibana/audit/test/test-audit-814.log-expected.json new file mode 100644 index 000000000000..9ab233fea606 --- /dev/null +++ b/filebeat/module/kibana/audit/test/test-audit-814.log-expected.json @@ -0,0 +1,171 @@ +[ + { + "@timestamp": "2023-06-19T15:18:47.298+00:00", + "event.action": "saved_object_create", + "event.category": [ + "database" + ], + "event.dataset": "kibana.audit", + "event.kind": "event", + "event.module": "kibana", + "event.outcome": "unknown", + "event.timezone": "-02:00", + "event.type": [ + "access" + ], + "fileset.name": "audit", + "input.type": "log", + "kibana.saved_object.id": "fleet-default-settings", + "kibana.saved_object.type": "ingest_manager_settings", + "labels.application": "elastic/fleet", + "log.level": "INFO", + "log.logger": "plugins.security.audit.ecs", + "log.offset": 0, + "message": "User is accessing ingest_manager_settings [id=fleet-default-settings]", + "process.pid": 7, + "service.node.roles": [ + "background_tasks", + "ui" + ], + "service.type": "kibana", + "trace.id": "809d3449277aba205a3ac539d23dbf7e", + "transaction.id": "49a38064b0f1dc1e" + }, + { + "@timestamp": "2023-06-19T15:18:48.987+00:00", + "event.action": "saved_object_create", + "event.category": [ + "database" + ], + "event.dataset": "kibana.audit", + "event.kind": "event", + "event.module": "kibana", + "event.outcome": "unknown", + "event.timezone": "-02:00", + "event.type": [ + "access" + ], + "fileset.name": "audit", + "input.type": "log", + "kibana.saved_object.id": "a09a5397-7b9a-5a73-a622-e29f4c635658", + "kibana.saved_object.type": "ingest-outputs", + "labels.application": "elastic/fleet", + "log.level": "INFO", + "log.logger": "plugins.security.audit.ecs", + "log.offset": 616, + "message": "User is accessing ingest-outputs [id=a09a5397-7b9a-5a73-a622-e29f4c635658]", + "process.pid": 7, + "service.node.roles": [ + "background_tasks", + "ui" + ], + "service.type": "kibana", + "trace.id": "809d3449277aba205a3ac539d23dbf7e", + "transaction.id": "49a38064b0f1dc1e" + }, + { + "@timestamp": "2023-06-19T15:18:53.426+00:00", + "event.action": "saved_object_create", + "event.category": [ + "database" + ], + "event.dataset": "kibana.audit", + "event.kind": "event", + "event.module": "kibana", + "event.outcome": "unknown", + "event.timezone": "-02:00", + "event.type": [ + "access" + ], + "fileset.name": "audit", + "input.type": "log", + "kibana.saved_object.id": "synthetics", + "kibana.saved_object.type": "epm-packages", + "labels.application": "elastic/fleet", + "log.level": "INFO", + "log.logger": "plugins.security.audit.ecs", + "log.offset": 1242, + "message": "User is accessing epm-packages [id=synthetics]", + "process.pid": 7, + "service.node.roles": [ + "background_tasks", + "ui" + ], + "service.type": "kibana", + "trace.id": "809d3449277aba205a3ac539d23dbf7e", + "transaction.id": "49a38064b0f1dc1e" + }, + { + "@timestamp": "2023-06-19T15:19:18.882+00:00", + "client.ip": "172.22.0.2", + "event.action": "http_request", + "event.category": [ + "web" + ], + "event.dataset": "kibana.audit", + "event.kind": "event", + "event.module": "kibana", + "event.outcome": "unknown", + "event.timezone": "-02:00", + "fileset.name": "audit", + "http.request.method": "get", + "input.type": "log", + "kibana.space_id": "default", + "log.level": "INFO", + "log.logger": "plugins.security.audit.ecs", + "log.offset": 1812, + "message": "User is requesting [/api/features] endpoint", + "process.pid": 7, + "related.user": [ + "elastic" + ], + "service.node.roles": [ + "background_tasks", + "ui" + ], + "service.type": "kibana", + "trace.id": "e2792f3f-4cf1-4f6d-b4eb-5b491724c295", + "transaction.id": "cf44f52888b9ec5a", + "url.domain": "kibana", + "url.path": "/api/features", + "url.port": 5601, + "url.scheme": "http", + "user.name": "elastic", + "user.roles": [ + "superuser" + ] + }, + { + "@timestamp": "2023-06-19T16:18:47.298+00:00", + "event.action": "saved_object_create", + "event.category": [ + "database" + ], + "event.dataset": "kibana.audit", + "event.kind": "event", + "event.module": "kibana", + "event.outcome": "unknown", + "event.timezone": "-02:00", + "event.type": [ + "access" + ], + "fileset.name": "audit", + "input.type": "log", + "kibana.saved_object.id": "abcde-fghijk", + "kibana.saved_object.type": "ingest_manager_settings", + "kibana.saved_object.name": "fleet-object-name", + "labels.application": "elastic/fleet", + "log.level": "INFO", + "log.logger": "plugins.security.audit.ecs", + "log.offset": 2466, + "message": "User is accessing ingest_manager_settings [id=fleet-default-settings]", + "process.pid": 7, + "service.node.roles": [ + "background_tasks", + "ui" + ], + "service.type": "kibana", + "trace.id": "809d3449277aba205a3ac539d23dbf7e", + "transaction.id": "49a38064b0f1dc1e" + } +] diff --git a/filebeat/module/kibana/fields.go b/filebeat/module/kibana/fields.go index 504d1f6283ec..fce968bbf78d 100644 --- a/filebeat/module/kibana/fields.go +++ b/filebeat/module/kibana/fields.go @@ -32,5 +32,5 @@ func init() { // AssetKibana returns asset data. // This is the base64 encoded zlib format compressed contents of module/kibana. func AssetKibana() string { - return "eJy8ls9ymzAQxu9+ih1ffCmM7fhffMilzaHT6a23TodZowVUC4lIi928fUfguGBj4rhpOe7ut/v7hJAIYEvPa9jKDWocALBkRWsY1oHhAMCSInS0htTnBbnYyoKl0Wt4GADAQQtfjSgVDQASSUq4dZULQGNOa3BkdzKmUBtBoTWKXJUG4OeC1p5hb6xoSY5If8pSa8riEOkA8U9NAYmxUKB1UqfwpeZTJnXhoa6J2MZ0ThodSXFMnUz6lhF8/gQmAc4ISkf2RQTonIklMgnYS86AM+mAdqQ5hEeMM08gNSAz5QWDJVcqduBDUGr5VNKxlRRhA4B+YV5Ub2UyvaPZfLEMaHW/CSZTcRfgbL4IZtPFYjKbLGfj8XjYUJ6vbsttgTH1eB16s1K8mK3K+1wOO5kFJVgqfgMW7khEZvOTYg59aS+fLzgSeiXUyhtA0WUbg1bciHr9Qv4V5mJ6P98IMQ7GOKZgMqFlsJrOk2CxnK1igUuxmq+ud4BCRGyi6t26Xn5H7A3Uu0AKB5whA7bd7NGBy9CSADYXDHwfHbbE6AOMcrRbYqnT0Y/roQUpYooSa/J3JbeUGx/yjf8ZPJackWYZo0eMCmt2UpDtddDWwIvmbPPg4Yzp2z8bdDKe3Mz76gd5ibX6UG8GvpnXEqq8F/hRoWMZO0IbZ6f4lbzqDPtMxhkkpUqkUiSuYtfIckfXwytjtmXxZuhadoD9P5zpWW3zdj6DfmgkoHElg5KaXPO2O72bm2MZU9dKXOJ8laBNkfr/BN887BzrGJneb+7H0lrSXLf1J1MN0j07J8bO0fW5dZKqg1E33MX+oaWnMCMUZLuXN1H+r0WT6GvhrmzxOwAA//+jaglP" + return "eJy8ls1y2kAMx+88hYYLl9oDhK9wyKXNodPprbdOxyO8sr1lvevsytC8fWdtQm0wDqFpOUr6Sz8tQiiALT2vYSs3qHEAwJIVrWFYG4YDAEuK0NEaUu8X5GIrC5ZGr+FhAAAHLXw1olQ0AEgkKeHWlS8AjTmtwZHdyZhCbQSF1ihylRuAnwtae4a9saIlOSL9CUutKYuDpQPEf2oKSIyFAq2TOoUvNZ8yqQsPcU3ENqZz0uhIiqPrpNK3jODzJzAJcEZQOrIvIkDnTCyRScBecgacSQe0I80hPGKceQKpAZkpLxgsuVKxA2+CUsunko6ppAgbAPQL86L6VibTO5rNF8uAVvebYDIVdwHO5otgNl0sJrPJcjYej4cN5fnrtrotMKaeXoe+WSlemq3C+7ocdjILSrBU/AYs3JGIzOYnxRz60F4+H3Ak9EqolTeAoss2Bq24EfX6h/wrzMX0fr4RYhyMcUzBZELLYDWdJ8FiOVvFApdiNV/d2IE39fbgA96ni/w5qBIEdYLrgVGIiE1UDaPrhXXEnrUeWykccIYM2AbfowOXoSUBbC6wfh8dZnj0AUY52i2x1Onox/XQghQxRYk1+buSW8qNN/nE/wweS85Is4zRI0aFNTspyPZ20NbAi+ZsTvCwFPtGZYNOxpObeV/dIJdYq81yM/DNvJZQ5b3Ajwody9gR2jg7xa/k9Q91n8k4g6RUiVSKxFXsGlnu6Hp4Zcy2LN4MXcsOsP+HMz2LbZ4TZ9APDQc0bghQUpNr/j2fHhPNsoypazkucb5K0KZI/WHjk4edZR0j0/vV/VhaS5rrtH4z1SDdtXNi7Cxd760TV22MuuEu5g8tPYUZoSDb/byJ8meWJtGXwl2Z4ncAAAD//zz3P1Q=" } diff --git a/go.mod b/go.mod index 935b4dd78a4f..7d5487267113 100644 --- a/go.mod +++ b/go.mod @@ -79,7 +79,7 @@ require ( github.com/elastic/go-seccomp-bpf v1.4.0 github.com/elastic/go-structform v0.0.10 github.com/elastic/go-sysinfo v1.13.1 - github.com/elastic/go-ucfg v0.8.6 + github.com/elastic/go-ucfg v0.8.7 github.com/elastic/gosigar v0.14.2 github.com/fatih/color v1.15.0 github.com/fearful-symmetry/gorapl v0.0.4 diff --git a/go.sum b/go.sum index 6f829114376e..5cfffb864f44 100644 --- a/go.sum +++ b/go.sum @@ -710,8 +710,8 @@ github.com/elastic/go-structform v0.0.10 h1:oy08o/Ih2hHTkNcRY/1HhaYvIp5z6t8si8gn github.com/elastic/go-structform v0.0.10/go.mod h1:CZWf9aIRYY5SuKSmOhtXScE5uQiLZNqAFnwKR4OrIM4= github.com/elastic/go-sysinfo v1.13.1 h1:U5Jlx6c/rLkR72O8wXXXo1abnGlWGJU/wbzNJ2AfQa4= github.com/elastic/go-sysinfo v1.13.1/go.mod h1:GKqR8bbMK/1ITnez9NIsIfXQr25aLhRJa7AfT8HpBFQ= -github.com/elastic/go-ucfg v0.8.6 h1:stUeyh2goTgGX+/wb9gzKvTv0YB0231LTpKUgCKj4U0= -github.com/elastic/go-ucfg v0.8.6/go.mod h1:4E8mPOLSUV9hQ7sgLEJ4bvt0KhMuDJa8joDT2QGAEKA= +github.com/elastic/go-ucfg v0.8.7 h1:/bKaN553LY3MsfEIz2XOEEs+tRw03TzJCARrnVPpOyc= +github.com/elastic/go-ucfg v0.8.7/go.mod h1:4E8mPOLSUV9hQ7sgLEJ4bvt0KhMuDJa8joDT2QGAEKA= github.com/elastic/go-windows v1.0.1 h1:AlYZOldA+UJ0/2nBuqWdo90GFCgG9xuyw9SYzGUtJm0= github.com/elastic/go-windows v1.0.1/go.mod h1:FoVvqWSun28vaDQPbj2Elfc0JahhPB7WQEGa3c814Ss= github.com/elastic/gopacket v1.1.20-0.20211202005954-d412fca7f83a h1:8WfL/X6fK11iyX5t3Dd9dDMMNqPfEZNc//JsWGIhEgQ= diff --git a/libbeat/outputs/kafka/config.go b/libbeat/outputs/kafka/config.go index 8fff8dad0d5c..3e2c836a06f7 100644 --- a/libbeat/outputs/kafka/config.go +++ b/libbeat/outputs/kafka/config.go @@ -22,6 +22,7 @@ import ( "fmt" "math" "math/rand" + "regexp" "strings" "time" @@ -31,6 +32,7 @@ import ( "github.com/elastic/beats/v7/libbeat/common/fmtstr" "github.com/elastic/beats/v7/libbeat/common/kafka" "github.com/elastic/beats/v7/libbeat/common/transport/kerberos" + "github.com/elastic/beats/v7/libbeat/management" "github.com/elastic/beats/v7/libbeat/outputs/codec" "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" @@ -77,6 +79,11 @@ type kafkaConfig struct { Sasl kafka.SaslConfig `config:"sasl"` EnableFAST bool `config:"enable_krb5_fast"` Queue config.Namespace `config:"queue"` + + // Currently only used for validation. Those values are later + // unpacked into temporary structs whenever they're necessary. + Topic string `config:"topic"` + Topics []any `config:"topics"` } type metaConfig struct { @@ -102,6 +109,11 @@ var compressionModes = map[string]sarama.CompressionCodec{ "snappy": sarama.CompressionSnappy, } +// validTopicRegExp is used to validate the topic contains only valid characters +// when running under Elastic-Agent. The regexp is taken from: +// https://github.com/apache/kafka/blob/a126e3a622f2b7142f3543b9dbee54b6412ba9d8/clients/src/main/java/org/apache/kafka/common/internals/Topic.java#L33 +var validTopicRegExp = regexp.MustCompile("^[a-zA-Z0-9._-]+$") + func defaultConfig() kafkaConfig { return kafkaConfig{ Hosts: nil, @@ -169,6 +181,24 @@ func (c *kafkaConfig) Validate() error { return fmt.Errorf("compression_level must be between 0 and 9") } } + + if c.Topic == "" && len(c.Topics) == 0 { + return errors.New("either 'topic' or 'topics' must be defined") + } + + // When running under Elastic-Agent we do not support dynamic topic + // selection, so `topics` is not supported and `topic` is treated as an + // plain string + if management.UnderAgent() { + if len(c.Topics) != 0 { + return errors.New("'topics' is not supported when running under Elastic-Agent") + } + + if !validTopicRegExp.MatchString(c.Topic) { + return fmt.Errorf("topic '%s' is invalid, it must match '[a-zA-Z0-9._-]'", c.Topic) + } + } + return nil } diff --git a/libbeat/outputs/kafka/config_test.go b/libbeat/outputs/kafka/config_test.go index 25c0c5dce996..2435b274f6e9 100644 --- a/libbeat/outputs/kafka/config_test.go +++ b/libbeat/outputs/kafka/config_test.go @@ -25,6 +25,7 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/internal/testutil" + "github.com/elastic/beats/v7/libbeat/management" "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" @@ -32,16 +33,18 @@ import ( func TestConfigAcceptValid(t *testing.T) { tests := map[string]mapstr.M{ - "default config is valid": mapstr.M{}, "lz4 with 0.11": mapstr.M{ "compression": "lz4", "version": "0.11", + "topic": "foo", }, "lz4 with 1.0": mapstr.M{ "compression": "lz4", "version": "1.0.0", + "topic": "foo", }, "Kerberos with keytab": mapstr.M{ + "topic": "foo", "kerberos": mapstr.M{ "auth_type": "keytab", "username": "elastic", @@ -52,6 +55,7 @@ func TestConfigAcceptValid(t *testing.T) { }, }, "Kerberos with user and password pair": mapstr.M{ + "topic": "foo", "kerberos": mapstr.M{ "auth_type": "password", "username": "elastic", @@ -67,7 +71,9 @@ func TestConfigAcceptValid(t *testing.T) { test := test t.Run(name, func(t *testing.T) { c := config.MustNewConfigFrom(test) - c.SetString("hosts", 0, "localhost") + if err := c.SetString("hosts", 0, "localhost"); err != nil { + t.Fatalf("could not set 'hosts' on config: %s", err) + } cfg, err := readConfig(c) if err != nil { t.Fatalf("Can not create test configuration: %v", err) @@ -89,13 +95,17 @@ func TestConfigInvalid(t *testing.T) { "realm": "ELASTIC", }, }, + // The default config does not set `topic` nor `topics`. + "No topics or topic provided": mapstr.M{}, } for name, test := range tests { test := test t.Run(name, func(t *testing.T) { c := config.MustNewConfigFrom(test) - c.SetString("hosts", 0, "localhost") + if err := c.SetString("hosts", 0, "localhost"); err != nil { + t.Fatalf("could not set 'hosts' on config: %s", err) + } _, err := readConfig(c) if err == nil { t.Fatalf("Can create test configuration from invalid input") @@ -104,6 +114,84 @@ func TestConfigInvalid(t *testing.T) { } } +func TestConfigUnderElasticAgent(t *testing.T) { + oldUnderAgent := management.UnderAgent() + t.Cleanup(func() { + // Restore the previous value + management.SetUnderAgent(oldUnderAgent) + }) + + management.SetUnderAgent(true) + + tests := []struct { + name string + cfg mapstr.M + expectError bool + }{ + { + name: "topic with all valid characters", + cfg: mapstr.M{ + "topic": "abcdefghijklmnopqrstuvxz-ABCDEFGHIJKLMNOPQRSTUVXZ_01234567890.", + }, + }, + { + name: "topics is provided", + cfg: mapstr.M{ + "topics": []string{"foo", "bar"}, + }, + expectError: true, + }, + { + name: "topic cannot contain invalid characters", + cfg: mapstr.M{ + "topic": "foo bar", + }, + expectError: true, + }, + { + name: "topic with invalid characters", + cfg: mapstr.M{ + "topic": "foo + bar", + }, + expectError: true, + }, + { + name: "topic with invalid characters from dynamic topic selection", + cfg: mapstr.M{ + "topic": "%{event.field}", + }, + expectError: true, + }, + + // The default config does not set `topic` not `topics`. + { + name: "empty config is invalid", + cfg: mapstr.M{}, + expectError: true, + }, + } + + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + c := config.MustNewConfigFrom(test.cfg) + if err := c.SetString("hosts", 0, "localhost"); err != nil { + t.Fatalf("could not set 'hosts' on config: %s", err) + } + + _, err := readConfig(c) + + if test.expectError && err == nil { + t.Fatalf("invalid configuration must not be created") + } + + if !test.expectError && err != nil { + t.Fatalf("could not create config: %s", err) + } + }) + } +} + func TestBackoffFunc(t *testing.T) { testutil.SeedPRNG(t) tests := map[int]backoffConfig{ @@ -178,6 +266,7 @@ func TestTopicSelection(t *testing.T) { for name, test := range cases { t.Run(name, func(t *testing.T) { + test := test selector, err := buildTopicSelector(config.MustNewConfigFrom(test.cfg)) if err != nil { t.Fatalf("Failed to parse configuration: %v", err) diff --git a/metricbeat/docs/modules/mysql.asciidoc b/metricbeat/docs/modules/mysql.asciidoc index 0c5a793a29ae..8711359bf5f6 100644 --- a/metricbeat/docs/modules/mysql.asciidoc +++ b/metricbeat/docs/modules/mysql.asciidoc @@ -89,6 +89,18 @@ metricbeat.modules: # By setting raw to true, all raw fields from the status metricset will be added to the event. #raw: false + + # Optional SSL/TLS. By default is false. + #ssl.enabled: true + + # List of root certificates for SSL/TLS server verification + #ssl.certificate_authorities: ["/etc/pki/root/ca.crt"] + + # Certificate for SSL/TLS client authentication + #ssl.certificate: "/etc/pki/client/cert.crt" + + # Client certificate key file + #ssl.key: "/etc/pki/client/cert.key" ---- [float] diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 6659ca292769..3f9ccb0a9dbc 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -764,6 +764,18 @@ metricbeat.modules: # By setting raw to true, all raw fields from the status metricset will be added to the event. #raw: false + # Optional SSL/TLS. By default is false. + #ssl.enabled: true + + # List of root certificates for SSL/TLS server verification + #ssl.certificate_authorities: ["/etc/pki/root/ca.crt"] + + # Certificate for SSL/TLS client authentication + #ssl.certificate: "/etc/pki/client/cert.crt" + + # Client certificate key file + #ssl.key: "/etc/pki/client/cert.key" + #--------------------------------- NATS Module --------------------------------- - module: nats metricsets: diff --git a/metricbeat/module/mysql/_meta/Dockerfile b/metricbeat/module/mysql/_meta/Dockerfile index 2051c726595e..b701ad617ea3 100644 --- a/metricbeat/module/mysql/_meta/Dockerfile +++ b/metricbeat/module/mysql/_meta/Dockerfile @@ -5,4 +5,8 @@ ENV MYSQL_ROOT_PASSWORD test HEALTHCHECK --interval=1s --retries=90 CMD mysql -u root -p$MYSQL_ROOT_PASSWORD -h$HOSTNAME -P 3306 -e "SHOW STATUS" > /dev/null +COPY /certs/root-ca.pem /etc/certs/root-ca.pem +COPY /certs/server-cert.pem /etc/certs/server-cert.pem +COPY /certs/server-key.pem /etc/certs/server-key.pem + COPY test.cnf /etc/mysql/conf.d/test.cnf diff --git a/metricbeat/module/mysql/_meta/certs/client-cert.pem b/metricbeat/module/mysql/_meta/certs/client-cert.pem new file mode 100755 index 000000000000..df9c76e08626 --- /dev/null +++ b/metricbeat/module/mysql/_meta/certs/client-cert.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDDDCCAfQCAQEwDQYJKoZIhvcNAQELBQAwSjELMAkGA1UEBhMCVVMxEzARBgNV +BAgMCkNhbGlmb3JuaWExFDASBgNVBAcMC1NhbnRhIENsYXJhMRAwDgYDVQQDDAdm +YWtlLUNBMB4XDTI0MDIxNTIzNTA0MloXDTMzMTIyNDIzNTA0MlowTjELMAkGA1UE +BhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExFDASBgNVBAcMC1NhbnRhIENsYXJh +MRQwEgYDVQQDDAtmYWtlLWNsaWVudDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC +AQoCggEBAIqHZbSUB1x/iW6DxaRlkFWjPuZ+F1wYTGvfpqnxZgZY1k5vSJTy3ETe +y3TelpEPBWEmsgHDx4bjuqeG+3my9dDEKEIYgXkfkfHREndVxPDfnRdfXPfp3qbm +wV2bdJnpSQzCg+lv8e8U+kMv0WcmwTuwlpVG0Rnb6vFdOs67/IIlBvI9sP5BKDYL +YFRxaoc8fLb8UMkfQ0BSmT4Rvmq5MSETh4re7OecV6pN0naEWhZf72mr/HiTAhb6 +xZJNSvNAzvdkQnhwt9aHemGQLRZD+4dduZYn27cwK4ySTZdyMoKn66HqMIfXPvr8 +LlICP4Gb8Df/JuUZVRbI13P+Xqujd8kCAwEAATANBgkqhkiG9w0BAQsFAAOCAQEA +gwA1+nZISC6QF9JtkOGrPpBZk6v1iy4iLsZSNaoinkB/FgesIpNrTFG0k6exSBV1 +pwQSmMVNSEsUOOjEq/Vk98014Kf8QVqfkdcujaBNPtxMqsocOO9Od78UuX5QdZXi +ayhkzrcPX4HTwjTqKFlJxb92rHrBx/GIWa68TeAjwbRiZmDASpVCEI2HnkBkFWTs +5Ux4wlC3JrnY3Jxb7QfDK94g9r5s1ljHeVki83cUYaI5TdY7F0uP+O6TvlhCPrjd +5708kRZJHnKThu3aE8HJYIbYhHocm9DszbnObd4SqECjfd6YNbREBhyaHJdCY/j2 +hm1zhBiW24dazs108uhFsQ== +-----END CERTIFICATE----- diff --git a/metricbeat/module/mysql/_meta/certs/client-key.pem b/metricbeat/module/mysql/_meta/certs/client-key.pem new file mode 100755 index 000000000000..33430372fd24 --- /dev/null +++ b/metricbeat/module/mysql/_meta/certs/client-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCKh2W0lAdcf4lu +g8WkZZBVoz7mfhdcGExr36ap8WYGWNZOb0iU8txE3st03paRDwVhJrIBw8eG47qn +hvt5svXQxChCGIF5H5Hx0RJ3VcTw350XX1z36d6m5sFdm3SZ6UkMwoPpb/HvFPpD +L9FnJsE7sJaVRtEZ2+rxXTrOu/yCJQbyPbD+QSg2C2BUcWqHPHy2/FDJH0NAUpk+ +Eb5quTEhE4eK3uznnFeqTdJ2hFoWX+9pq/x4kwIW+sWSTUrzQM73ZEJ4cLfWh3ph +kC0WQ/uHXbmWJ9u3MCuMkk2XcjKCp+uh6jCH1z76/C5SAj+Bm/A3/yblGVUWyNdz +/l6ro3fJAgMBAAECggEAEPRCAHQrA/k4c9oFBQoonHCMrNdDCuKO7NdsHYm1ucJi +5SnVxWQFTRkC59hrr1B6MTIUEGb6iyHhOOpqafI7B0xQnIlFBFLWsPSseMY6opvN +jTwew9k/xqfAg/E4F7OvXPRMAnSQ1LjZqcInE+Owe9qQjW/DvPFXS2fEgCOOA4vw +M6w6USf8UTsXBzMvRnDHMTQM0vfKNNSdopYDPeQc4YQ1A2AjkpYXZVWXFcFsE9zw +xFVZ9k6tP+gzk6shJjsbBoQ7qWwhdq1Q5tJ28FTaCVXDAp8l6yIFuZuI7r23O7+0 +ngxSejABJ3m9NmG0J7DPGU6zXhJW5nylWcSk5vwMkQKBgQDCWIRe4iSW0eGYBSe5 +hBoQgLe7aMAbsaCrHjTYQkKvI25YlfJ08OVU7oB/Bng/9AlpJlouGz67/W0PiRaz +jlP370p92IiwehUl9PkuVDpex4l2rDLCM1iVrPbxhbm/7+2nro2M/0/4iUyIK+Gr +Rpcqj2dQ3qarD+UmLXYPOoyRuQKBgQC2ec0sWyU67QuFaTemvTH8YFu68BfQqg6t +YQMc4+wj30ww0TZHFYVwyvR4agTOdFjwIUTERRN3EcFmlV5x+fGz/LfUdVYJj8B0 +lXakqeATsGJHngrdlyM+m+g+6JI1SUTshMa/xXVAUx8NZESOVE5JeZH6TD4/9Q3y +ijtithtekQKBgQCPeso/QrXAozLqCORLEjwr8tuygKNTzs/PhX1+K20P4BiXThyy +OScWjP5QyXX9wS0xdB8f6v1lzLO3xH3+EhXr9b4JKtO/dmImo7VTftuZHbde5cKT +nVTJK+kkZpW8HmZWZYgbkGJ6GuNlpP/2cycnRLgB/F8P66xBg06l75PYAQKBgGap +GhR1ZvnC+TNiocuuL5wkfhcrEsrzkfRbWwv68xSvgUcJvTa61etCU84XH4MjlBHt +NaoSjsPzelKDgLIxA5nWeXoPVYtlk8pDeI9lf0q0dmaCdOx8JnkH797Mq81M3nkO +rl6f8bpxyUuYeLV2muDdg5JFKNSEwwcMXCLJ/5XxAoGAKIkS02jWudDoBzubdFe/ +c5jSYufTZOmErnjnSKGkj9oZGVP6RYDhkHMPOxadO/4OLOKo6Phkg9yRmPG2tyKA ++ddgYP7zXEnsLxrjumoYTvcWgs1AHUUH4kA5SdImzYbSSfPW5h0KkvB+gYaukBGa +XHILry/59LkxU+nP1ZCVvt8= +-----END PRIVATE KEY----- diff --git a/metricbeat/module/mysql/_meta/certs/client-req.pem b/metricbeat/module/mysql/_meta/certs/client-req.pem new file mode 100755 index 000000000000..3295c803f8d3 --- /dev/null +++ b/metricbeat/module/mysql/_meta/certs/client-req.pem @@ -0,0 +1,16 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIICkzCCAXsCAQAwTjELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWEx +FDASBgNVBAcMC1NhbnRhIENsYXJhMRQwEgYDVQQDDAtmYWtlLWNsaWVudDCCASIw +DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAIqHZbSUB1x/iW6DxaRlkFWjPuZ+ +F1wYTGvfpqnxZgZY1k5vSJTy3ETey3TelpEPBWEmsgHDx4bjuqeG+3my9dDEKEIY +gXkfkfHREndVxPDfnRdfXPfp3qbmwV2bdJnpSQzCg+lv8e8U+kMv0WcmwTuwlpVG +0Rnb6vFdOs67/IIlBvI9sP5BKDYLYFRxaoc8fLb8UMkfQ0BSmT4Rvmq5MSETh4re +7OecV6pN0naEWhZf72mr/HiTAhb6xZJNSvNAzvdkQnhwt9aHemGQLRZD+4dduZYn +27cwK4ySTZdyMoKn66HqMIfXPvr8LlICP4Gb8Df/JuUZVRbI13P+Xqujd8kCAwEA +AaAAMA0GCSqGSIb3DQEBCwUAA4IBAQBr6+WE3t0KdMpEBBC81IUHkXNB9Mf5EYKG +d1ev6jq1bi2jw6WqAGbqYp1W0awEjZJZcS2skXoy8QIFDNjznHPgKEXB9b98nj34 +TLpszCrlcQteWmzRCspwkhdrXNGE4Z4UMgN+xoh2P/dujK4kGH6HFcF1Fo4ajDUX +HT5vybjQuQlPDgt6Ufs+Pjotr5uCzLbIsFN1QG6gKVY90WAzPsa0XYN1ehMpkLsM +8vbVP0uRT6/VXTenbTtqqQ5Y70gmeiF/EssnQ9rM3vkGUW1A/9j23agLmlOVaCWw +HSN5HqrFUIlsLFIDDTgi7icW4Uk+7qdMSF7ooMOJIm27PGc49u4U +-----END CERTIFICATE REQUEST----- diff --git a/metricbeat/module/mysql/_meta/certs/root-ca-key.pem b/metricbeat/module/mysql/_meta/certs/root-ca-key.pem new file mode 100755 index 000000000000..2343e39b1499 --- /dev/null +++ b/metricbeat/module/mysql/_meta/certs/root-ca-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDSrYQWHfzCy/+N +Nzbg03pW/xnJ4wsEOeSuaYJfrkzU87pwlOFY57bhxe+3rektDOOziwjCxo4a8rKu +YmHRYKx4XdBtTjPnimRiwymSnemZdABWLNuJyvWen6iNJQqrcSesvobAtaQ265A9 +faRPn/Hjx5CH5x52hLhcpo6Yg4Ae6K2dnGbahFb1DI7Btfcf+PYiUau5DRiJiIpU +9K9hBbPmPuo0hsGiAYCJkTspdDMrFsBA6hNadamzsXy6AzB82Pu19nckR20kJVlG +Ioebg6mlHlcTV1qCsiWZBR/ghGGNHBp15EIXvIDpEJ4rcuy4AER4lXIdpG2RPD7Y ++Y7EGi0zAgMBAAECggEAU/SCuR+dHPGXdqqEq4aK78U7/SiFuNrrRxfTtRZsFsbD +yt6BiODaD9HFrCBZVjMXQHLM/HWMpq+Fxhl9mqcYQ+U6zHxIEeKkC3lzhTJ5p0XD +ZpP8rsYbKGm+jPSwck6m/V91qrEX7izkb6S0iGiYR+m8rnPLP3a3U3CqTZvFwErG +n7jk7caLZcT9+p7/TLlDIyx4ha4+7RRaL9OC1dNH8ADOkSHk/vaE6aU8J8PJ4YZg +QvNfsuo7FtDMq3OIkMAsHseuX90X8c3ZS7lNdCTRU7YuC1+8+l6xGs1Arjv1jqnd +9gIo6kh88Ng8zi4TkGLVAnfc55eXmB+f7PPN93fMeQKBgQD0uqDSsvPNnaY6DJIF +Gyz4qExyYH/h2QFT5M4bb0hQNIkP187JhBZw8Et2AvBtSBhs8dXfBxu736KRs8XG +b60iw2qXqo1XUEUO7R0VMO6NcA8Hk206X+p7ukn5RExzv2MurD+3f8QM8CypFA57 +UnSWdDCrOAh6WU5zfcz9woOM2QKBgQDcYWvqbV8XoyhJBG5PkG3UzYtOi/Es/fGH +qt03ZyyfYdCbhAknqftuj6ZzlMfVV3xOSXX+sdr0rLzammnRdlPJtJfjk8lUYa/i +0hy4eTHm7o1iZJfMS9rCMH9uTwyNGnb67u8kW16BuzaLbJMtd7IKtEG69U63abZX +t+zqmxGy6wKBgQCD43w+cNCxdA+cYx/ifpXK4DBqx5TDq0Zq5vkokd1/1AA1uJEp +yvSpIucYD1dxHZSESgR/sH4Czu/249JnMdI11OjCGdkYQBsngyPUQs2dDdIbvBj2 +h7B/w5KQMn2dN3yFL7Ea/FE0w87dxABV98b7OlzsOUNgZHbCCP8LluN8aQKBgGS3 +RTly2JWV5DBSjRNhn0A026h+/i6gs8RbyxOp3FPOwSaBlimBXr4telWyNg2DGPUy +T3Gh2L4fP4PsM9YdbLdvCEdiYA1nQ5m2ipeoE61Fcmn4LQOZ2xUKUwKXr9XAtYWC +stn7w9ooNApOCYkq/bw0myGVQG9EKag3D1g8nD8XAoGAZLJlDhlfFaWa7jy1VF/g +JWcsN/+BfTjBY6t3npxzg4pdi7lHhuAZ45PLnQMTIdWCkqgigt224kcbUy3b351u +lzoSiLatNXj5Q3on85ZNRaOMLqp0ueIzOLWvC+CRp46wXlwxTrPxghXatUBPsG47 +mO/mtw9gmaJ8UBW/SuxS24g= +-----END PRIVATE KEY----- diff --git a/metricbeat/module/mysql/_meta/certs/root-ca.pem b/metricbeat/module/mysql/_meta/certs/root-ca.pem new file mode 100755 index 000000000000..9b3e4f60fe8e --- /dev/null +++ b/metricbeat/module/mysql/_meta/certs/root-ca.pem @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDdTCCAl2gAwIBAgIUUp8x6W/bui3FjHLnJfIb7AsKBIwwDQYJKoZIhvcNAQEL +BQAwSjELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExFDASBgNVBAcM +C1NhbnRhIENsYXJhMRAwDgYDVQQDDAdmYWtlLUNBMB4XDTI0MDIxNTIzNTAzNVoX +DTMzMTIyNDIzNTAzNVowSjELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3Ju +aWExFDASBgNVBAcMC1NhbnRhIENsYXJhMRAwDgYDVQQDDAdmYWtlLUNBMIIBIjAN +BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA0q2EFh38wsv/jTc24NN6Vv8ZyeML +BDnkrmmCX65M1PO6cJThWOe24cXvt63pLQzjs4sIwsaOGvKyrmJh0WCseF3QbU4z +54pkYsMpkp3pmXQAVizbicr1np+ojSUKq3EnrL6GwLWkNuuQPX2kT5/x48eQh+ce +doS4XKaOmIOAHuitnZxm2oRW9QyOwbX3H/j2IlGruQ0YiYiKVPSvYQWz5j7qNIbB +ogGAiZE7KXQzKxbAQOoTWnWps7F8ugMwfNj7tfZ3JEdtJCVZRiKHm4OppR5XE1da +grIlmQUf4IRhjRwadeRCF7yA6RCeK3LsuABEeJVyHaRtkTw+2PmOxBotMwIDAQAB +o1MwUTAdBgNVHQ4EFgQURA7Q9JPfB4mveB0vzmoqNJ2HSZUwHwYDVR0jBBgwFoAU +RA7Q9JPfB4mveB0vzmoqNJ2HSZUwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0B +AQsFAAOCAQEAB4NGJFZpzltHLqvInSU/EQxdIHgifihOFzsXTEXkdrmkfEw5puVL +fzg6qnLOunh3GAwLCnM0aIzDLS8WAS509Jwwidn7OtBpYV+jIzJrrTycWjAdvcHC +WToPTueXxwaAD3pCrus0w9H8egoQ1haNVmQm0OWcv3My82cNbZwViuQSCrky1srL +N5l7UM0gbXKeZjTGHIoTIjQJDgJT8PydsxpOZq7CcKRDBdF5nYMcUq8wltneb0Nh +7DuLLdxEM11XzIRT4GLRxT2xqwW7UpLfWpuo+niCvmNFY6SzyHFR1vFI3Kw1rYXh +3cbEtHtRvcNQg6Jp/zoHDcXMS3hDMeN2vQ== +-----END CERTIFICATE----- diff --git a/metricbeat/module/mysql/_meta/certs/server-cert.pem b/metricbeat/module/mysql/_meta/certs/server-cert.pem new file mode 100755 index 000000000000..1ca56e3f44f9 --- /dev/null +++ b/metricbeat/module/mysql/_meta/certs/server-cert.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDDDCCAfQCAQEwDQYJKoZIhvcNAQELBQAwSjELMAkGA1UEBhMCVVMxEzARBgNV +BAgMCkNhbGlmb3JuaWExFDASBgNVBAcMC1NhbnRhIENsYXJhMRAwDgYDVQQDDAdm +YWtlLUNBMB4XDTI0MDIxNTIzNTAzOFoXDTMzMTIyNDIzNTAzOFowTjELMAkGA1UE +BhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWExFDASBgNVBAcMC1NhbnRhIENsYXJh +MRQwEgYDVQQDDAtmYWtlLXNlcnZlcjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC +AQoCggEBAMuPqkUt/Ax9s/h5LPxXU0m6OAEp1InLbR6x//hGVgmIiQu5/Fg1VfmZ +YbwraXxs4JDfMUyK6bd/bk2o71I1pnLmoFmQvawDRxOqkA1NLpF2FJtk0eevkF1D +crC9T1SfrzlwrucqqUXowdprVXFFVbFQTXsSyD8Nv/MGzDgmDtmMXQ8sLVqjGIEM +akuPMbNCVNTVnd/53WMaDzopnam/NCJNDGp2RVhf+KuOWLTURXFYN6j1z+f/1BNa +4QW+WtofzYkAWEcvCc8zeXUhwL6xE5gDyq1NkQ/ejqQq+iIJLd1FUFOH1jPSgmW5 +3CiWih2Is6VA0hCzDirdFtAHTui/OekCAwEAATANBgkqhkiG9w0BAQsFAAOCAQEA +vdGGVxaeSEfOkx+D7uYCx0blnobJoclggQP3fOIpyrU/LCeka+F8dvFvuGJLvn3A +JOMZZHCVnK6jjJYHFeolRCxd9iULYHD+dkWDr6uhvNMfwIt7UzUmtbznHAaD+ays +X0H70Z9+jmr3uFkevRbFkvDZqzdRYi/12oPM+0Skra3ouYen6zAtPU0Hruc0jyBP +W7V6mMSmCUPKTOJRZgDEIEBvu43rwEbQUG0ayqF1sLv+D6hjFrFJ2gCxgVH/+C9E +h0NF2Kdpb+jECCu3yhQA536Ugi9k96zJqJonu9jP4ODXMTG2qmsdFFW1zyFb9DbV +bjUsiDE7bEumHY2NEfzr3A== +-----END CERTIFICATE----- diff --git a/metricbeat/module/mysql/_meta/certs/server-key.pem b/metricbeat/module/mysql/_meta/certs/server-key.pem new file mode 100755 index 000000000000..d1a7d286a1c3 --- /dev/null +++ b/metricbeat/module/mysql/_meta/certs/server-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDLj6pFLfwMfbP4 +eSz8V1NJujgBKdSJy20esf/4RlYJiIkLufxYNVX5mWG8K2l8bOCQ3zFMium3f25N +qO9SNaZy5qBZkL2sA0cTqpANTS6RdhSbZNHnr5BdQ3KwvU9Un685cK7nKqlF6MHa +a1VxRVWxUE17Esg/Db/zBsw4Jg7ZjF0PLC1aoxiBDGpLjzGzQlTU1Z3f+d1jGg86 +KZ2pvzQiTQxqdkVYX/irjli01EVxWDeo9c/n/9QTWuEFvlraH82JAFhHLwnPM3l1 +IcC+sROYA8qtTZEP3o6kKvoiCS3dRVBTh9Yz0oJludwoloodiLOlQNIQsw4q3RbQ +B07ovznpAgMBAAECggEADLAux9Me89ReBG3hLPVwfpb56LCny9L/QTuNHfecY0m8 +aRu1q/XfHwi9e9Ik6BmNQdp3ozLBcKujv3l5OWGYt27CrfKEsBUgOAyYoAugjHaU +wD7fipZ55CZRHs0eBcNSU70/Wa9iD7Z7Ztbr43yT49KCkdpQ2wVLYqWY0yMkJ9Eo +ZUJ8fL+yDMeJxnhQSIejK62TQI3FdMz+aNXA6AO0YiSfqagTS8GVNZQvZzvyxYS0 +DpiydzKSbS2RXkf3waClU5hDGwqhNxXa9bya/KrLvm4ag/VaV0O1M9jwFOKwfUGY +0SDELz/mxsOmGntTUbtuH7VSvnqkJHfACUcNkkIjAQKBgQD5pwIzrPnGrljDcFqu +OCRxhiRjgCNth4ObBbmj2n0BV5Uw33o1VlN/+GCfKcIQ1+tHOUrEtkwP5mMatUbf +4G4K/+bO3eWAf+ia5hkSVASbU0ui36iSkPWLYJr0oDx0N6Vw+ZK7oxqLGqW2dm4Y +Q1TFaIDd2wUGPYAuDaqPDHecCQKBgQDQvKXy9Ueh4iTbz3sH6Kp4wGN2BsjWWOVn +Hi4QoqnDoLrguhCe5vvNyxfayziu9hUKzP8kBHQOY/2xpKv+epPuw6hgaD0Mnh/w +UcWEqZs102y0zZcQISfG8TUoLHW31T87veB3YEVIB+8uZg1CWJ7aDKe8UmugVGV2 +k2sMG7fm4QKBgHq0z6w+lPZGs3I8QxXmmmMCH9iYHtGzDcigY8JZnZ+PQNEoxpR4 +vcnkdvlEORK2TfpP+qP9Rh16i7OQ7ikT0oKtjPCYuDkUpWudNS2BBlKh+kcvz1da +0JWVAhTCvXQR9cs1oB2B6YX9rv2j8DEUxxHQb6acBDgw+lOoe/CbnB6hAoGBAKxg +bcbjCcHFCF1BzT8tw8GuVzS7y5U/mkp64N26BunXzRwSa/FdnOpI4q07j9bkv2HJ +ApZS2yibKIFQFP01av8NMvpSer/1wThrvuqcSeG8dJQnB645QykGPrirZpdmki6a +0kijBvPCIaI2gpKcrqoxMz/Q7LJdn+C5Qvif11HhAoGAfai8GYFiXuShvf+8gjOt +qIsBMV3YexcX11qPD5vVJMCW1xLbpb9f3sPf8P31TB8tz5JA3aG24k8yURtgTA4Z +2I6Jo9vwMjAdOxHTalqMllDvBj5S5+cX38kGdcxdcbAiUHwIoXy6cjcGbeO/SesR +L1bbyZA45gpsWFxFr5V67G0= +-----END PRIVATE KEY----- diff --git a/metricbeat/module/mysql/_meta/certs/server-req.pem b/metricbeat/module/mysql/_meta/certs/server-req.pem new file mode 100755 index 000000000000..035ab7e2faf3 --- /dev/null +++ b/metricbeat/module/mysql/_meta/certs/server-req.pem @@ -0,0 +1,16 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIICkzCCAXsCAQAwTjELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWEx +FDASBgNVBAcMC1NhbnRhIENsYXJhMRQwEgYDVQQDDAtmYWtlLXNlcnZlcjCCASIw +DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMuPqkUt/Ax9s/h5LPxXU0m6OAEp +1InLbR6x//hGVgmIiQu5/Fg1VfmZYbwraXxs4JDfMUyK6bd/bk2o71I1pnLmoFmQ +vawDRxOqkA1NLpF2FJtk0eevkF1DcrC9T1SfrzlwrucqqUXowdprVXFFVbFQTXsS +yD8Nv/MGzDgmDtmMXQ8sLVqjGIEMakuPMbNCVNTVnd/53WMaDzopnam/NCJNDGp2 +RVhf+KuOWLTURXFYN6j1z+f/1BNa4QW+WtofzYkAWEcvCc8zeXUhwL6xE5gDyq1N +kQ/ejqQq+iIJLd1FUFOH1jPSgmW53CiWih2Is6VA0hCzDirdFtAHTui/OekCAwEA +AaAAMA0GCSqGSIb3DQEBCwUAA4IBAQAK3+eAfReXoGP3CQvTE/Bd6u+u5kG65stV +DONrBzhMQ4R36X+Q6q65qJ0rmvwZcUfkIauQzdNv9ZfCDT7pO1VtNT0R+H6+shz9 +JiwGOudAlFSt31Ps0+lDm6WjA6J1Nmr9N7XrsmfdW4z2n1UZSPS9mOZIj+PpUtQw +OzIwJ/+btS/RVO0cGGFkoFwhrYKilAbq+SsMxMVxPcXUP+xLFYn6FCNFbf5uBpLz +ZM7HBDh2uVfwsaptnY3v+EIELCsXsFm9uj4zG45fJmu4KARY6FAi9sEvfA1ieZuU +8hmovXhKq6eSU2fPoeurRV1gxuanuFObd39LRoCTy3fCnqTZFxXg +-----END CERTIFICATE REQUEST----- diff --git a/metricbeat/module/mysql/_meta/config.reference.yml b/metricbeat/module/mysql/_meta/config.reference.yml index 03880a5ad6ab..4e5cc470aca1 100644 --- a/metricbeat/module/mysql/_meta/config.reference.yml +++ b/metricbeat/module/mysql/_meta/config.reference.yml @@ -21,3 +21,15 @@ # By setting raw to true, all raw fields from the status metricset will be added to the event. #raw: false + + # Optional SSL/TLS. By default is false. + #ssl.enabled: true + + # List of root certificates for SSL/TLS server verification + #ssl.certificate_authorities: ["/etc/pki/root/ca.crt"] + + # Certificate for SSL/TLS client authentication + #ssl.certificate: "/etc/pki/client/cert.crt" + + # Client certificate key file + #ssl.key: "/etc/pki/client/cert.key" diff --git a/metricbeat/module/mysql/_meta/config.yml b/metricbeat/module/mysql/_meta/config.yml index 367b32e91736..a86258fca3b5 100644 --- a/metricbeat/module/mysql/_meta/config.yml +++ b/metricbeat/module/mysql/_meta/config.yml @@ -18,3 +18,15 @@ # Password of hosts. Empty by default. #password: secret + + # Optional SSL/TLS. By default is false. + #ssl.enabled: true + + # List of root certificates for SSL/TLS server verification + #ssl.certificate_authorities: ["/etc/pki/root/ca.crt"] + + # Certificate for SSL/TLS client authentication + #ssl.certificate: "/etc/pki/client/cert.crt" + + # Client certificate key file + #ssl.key: "/etc/pki/client/cert.key" \ No newline at end of file diff --git a/metricbeat/module/mysql/_meta/test.cnf b/metricbeat/module/mysql/_meta/test.cnf index f759a49631d3..24eec52dd05f 100644 --- a/metricbeat/module/mysql/_meta/test.cnf +++ b/metricbeat/module/mysql/_meta/test.cnf @@ -1,2 +1,6 @@ [mysqld] bind-address = 0.0.0.0 +require_secure_transport = OFF +ssl-ca = /etc/certs/root-ca.pem +ssl-cert = /etc/certs/server-cert.pem +ssl-key = /etc/certs/server-key.pem \ No newline at end of file diff --git a/metricbeat/module/mysql/config.go b/metricbeat/module/mysql/config.go new file mode 100644 index 000000000000..96704bef4799 --- /dev/null +++ b/metricbeat/module/mysql/config.go @@ -0,0 +1,32 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mysql + +import ( + "crypto/tls" + + "github.com/elastic/elastic-agent-libs/transport/tlscommon" +) + +type Config struct { + Hosts []string `config:"hosts" validate:"required"` + Username string `config:"username"` + Password string `config:"password"` + TLS *tlscommon.Config `config:"ssl"` + TLSConfig *tls.Config +} diff --git a/metricbeat/module/mysql/docker-compose.yml b/metricbeat/module/mysql/docker-compose.yml index e112587fccdc..0644d9568ad1 100644 --- a/metricbeat/module/mysql/docker-compose.yml +++ b/metricbeat/module/mysql/docker-compose.yml @@ -2,10 +2,10 @@ version: '2.3' services: mysql: - image: docker.elastic.co/integrations-ci/beats-mysql:${MYSQL_VARIANT:-mysql}-${MYSQL_VERSION:-5.7.12}-1 + image: docker.elastic.co/integrations-ci/beats-mysql:${MYSQL_VARIANT:-mysql}-${MYSQL_VERSION:-8.0}-1 build: context: ./_meta args: - MYSQL_IMAGE: ${MYSQL_VARIANT:-mysql}:${MYSQL_VERSION:-5.7.12} + MYSQL_IMAGE: ${MYSQL_VARIANT:-mysql}:${MYSQL_VERSION:-8.0} ports: - 3306 diff --git a/metricbeat/module/mysql/galera_status/status.go b/metricbeat/module/mysql/galera_status/status.go index d1dc68cd0a25..6f27b8d4e8fc 100644 --- a/metricbeat/module/mysql/galera_status/status.go +++ b/metricbeat/module/mysql/galera_status/status.go @@ -42,7 +42,7 @@ func init() { // MetricSet for fetching Galera-MySQL server status type MetricSet struct { - mb.BaseMetricSet + *mysql.Metricset db *sql.DB } @@ -50,7 +50,13 @@ type MetricSet struct { // Loads query_mode config setting from the config file func New(base mb.BaseMetricSet) (mb.MetricSet, error) { cfgwarn.Experimental("The galera_status metricset is experimental.") - return &MetricSet{BaseMetricSet: base}, nil + + ms, err := mysql.NewMetricset(base) + if err != nil { + return nil, err + } + + return &MetricSet{Metricset: ms, db: nil}, nil } // Fetch methods implements the data gathering and data conversion to the right format @@ -58,7 +64,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { if m.db == nil { var err error - m.db, err = mysql.NewDB(m.HostData().URI) + m.db, err = mysql.NewDB(m.HostData().URI, m.Metricset.Config.TLSConfig) if err != nil { return fmt.Errorf("Galera-status fetch failed: %w", err) } diff --git a/metricbeat/module/mysql/mysql.go b/metricbeat/module/mysql/mysql.go index 35388a9a1bde..23c0f8dda106 100644 --- a/metricbeat/module/mysql/mysql.go +++ b/metricbeat/module/mysql/mysql.go @@ -21,14 +21,18 @@ Package mysql is Metricbeat module for MySQL server. package mysql import ( + "crypto/tls" "database/sql" "fmt" "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/elastic-agent-libs/transport/tlscommon" "github.com/go-sql-driver/mysql" ) +const TLSConfigKey = "custom" + func init() { // Register the ModuleFactory function for the "mysql" module. if err := mb.Registry.AddModule("mysql", NewModule); err != nil { @@ -38,16 +42,37 @@ func init() { func NewModule(base mb.BaseModule) (mb.Module, error) { // Validate that at least one host has been specified. - config := struct { - Hosts []string `config:"hosts" validate:"required"` - }{} - if err := base.UnpackConfig(&config); err != nil { + var c Config + if err := base.UnpackConfig(&c); err != nil { return nil, err } return &base, nil } +type Metricset struct { + mb.BaseMetricSet + Config Config +} + +func NewMetricset(base mb.BaseMetricSet) (*Metricset, error) { + var c Config + if err := base.Module().UnpackConfig(&c); err != nil { + return nil, fmt.Errorf("could not read config: %w", err) + } + + if c.TLS.IsEnabled() { + tlsConfig, err := tlscommon.LoadTLSConfig(c.TLS) + if err != nil { + return nil, fmt.Errorf("could not load provided TLS configuration: %w", err) + } + + c.TLSConfig = tlsConfig.ToConfig() + } + + return &Metricset{Config: c, BaseMetricSet: base}, nil +} + // ParseDSN creates a DSN (data source name) string by parsing the host. // It validates the resulting DSN and returns an error if the DSN is invalid. // @@ -55,9 +80,11 @@ func NewModule(base mb.BaseModule) (mb.Module, error) { // Example: root:test@tcp(127.0.0.1:3306)/ func ParseDSN(mod mb.Module, host string) (mb.HostData, error) { c := struct { - Username string `config:"username"` - Password string `config:"password"` + Username string `config:"username"` + Password string `config:"password"` + TLS *tlscommon.Config `config:"ssl"` }{} + if err := mod.UnpackConfig(&c); err != nil { return mb.HostData{}, err } @@ -86,6 +113,10 @@ func ParseDSN(mod mb.Module, host string) (mb.HostData, error) { noCredentialsConfig.User = "" noCredentialsConfig.Passwd = "" + if c.TLS.IsEnabled() { + config.TLSConfig = TLSConfigKey + } + return mb.HostData{ URI: config.FormatDSN(), SanitizedURI: noCredentialsConfig.FormatDSN(), @@ -99,10 +130,18 @@ func ParseDSN(mod mb.Module, host string) (mb.HostData, error) { // must be valid, otherwise an error will be returned. // // DSN Format: [username[:password]@][protocol[(address)]]/ -func NewDB(dsn string) (*sql.DB, error) { +func NewDB(dsn string, tlsConfig *tls.Config) (*sql.DB, error) { + if tlsConfig != nil { + err := mysql.RegisterTLSConfig(TLSConfigKey, tlsConfig) + if err != nil { + return nil, fmt.Errorf("registering custom tls config failed: %w", err) + } + } + db, err := sql.Open("mysql", dsn) if err != nil { return nil, fmt.Errorf("sql open failed: %w", err) } + return db, nil } diff --git a/metricbeat/module/mysql/mysql_integration_test.go b/metricbeat/module/mysql/mysql_integration_test.go index 5713a5821490..2fc96475646e 100644 --- a/metricbeat/module/mysql/mysql_integration_test.go +++ b/metricbeat/module/mysql/mysql_integration_test.go @@ -20,6 +20,9 @@ package mysql import ( + "crypto/tls" + "crypto/x509" + "os" "testing" "github.com/stretchr/testify/assert" @@ -31,9 +34,58 @@ import ( func TestNewDB(t *testing.T) { service := compose.EnsureUp(t, "mysql") - db, err := NewDB(GetMySQLEnvDSN(service.Host())) + db, err := NewDB(GetMySQLEnvDSN(service.Host()), nil) assert.NoError(t, err) err = db.Ping() assert.NoError(t, err) } + +func loadTLSConfig(caCertPath, clientCertPath, clientKeyPath string) (*tls.Config, error) { + caCert, err := os.ReadFile(caCertPath) + if err != nil { + return nil, err + } + caCertPool := x509.NewCertPool() + caCertPool.AppendCertsFromPEM(caCert) + + cert, err := tls.LoadX509KeyPair(clientCertPath, clientKeyPath) + if err != nil { + return nil, err + } + + tlsConfig := &tls.Config{ + Certificates: []tls.Certificate{cert}, + RootCAs: caCertPool, + MinVersion: tls.VersionTLS12, + } + + return tlsConfig, nil +} + +func TestNewDBWithSSL(t *testing.T) { + service := compose.EnsureUp(t, "mysql") + + tlsConfig, err := loadTLSConfig("_meta/certs/root-ca.pem", "_meta/certs/client-cert.pem", "_meta/certs/client-key.pem") + tlsConfig.InsecureSkipVerify = true + assert.NoError(t, err) + + db, err := NewDB(GetMySQLEnvDSN(service.Host())+"?tls=custom", tlsConfig) + assert.NoError(t, err) + + err = db.Ping() + assert.NoError(t, err) + + // Check if the current connection is using SSL + var sslCipher, variableName, value string + err = db.QueryRow(`show status like 'Ssl_cipher'`).Scan(&variableName, &sslCipher) + assert.NoError(t, err) + + // If sslCipher is not empty, then SSL is being used for the connection + assert.NotEmpty(t, variableName) + assert.NotEmpty(t, sslCipher) + + err = db.QueryRow(`show variables like 'have_ssl'`).Scan(&variableName, &value) + assert.NoError(t, err) + assert.Equal(t, "YES", value) +} diff --git a/metricbeat/module/mysql/query/query.go b/metricbeat/module/mysql/query/query.go index 35881d764016..d7bbaaa4cd7b 100644 --- a/metricbeat/module/mysql/query/query.go +++ b/metricbeat/module/mysql/query/query.go @@ -25,13 +25,17 @@ package query import ( "context" + "crypto/tls" "fmt" + mysqlDriver "github.com/go-sql-driver/mysql" + "github.com/elastic/beats/v7/libbeat/common/cfgwarn" "github.com/elastic/beats/v7/metricbeat/helper/sql" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/module/mysql" "github.com/elastic/elastic-agent-libs/mapstr" + "github.com/elastic/elastic-agent-libs/transport/tlscommon" ) func init() { @@ -57,8 +61,10 @@ type MetricSet struct { mb.BaseMetricSet db *sql.DbClient Config struct { - Queries []query `config:"queries" validate:"nonzero,required"` - Namespace string `config:"namespace" validate:"nonzero,required"` + Queries []query `config:"queries" validate:"nonzero,required"` + Namespace string `config:"namespace" validate:"nonzero,required"` + TLS *tlscommon.Config `config:"ssl"` + TLSConfig *tls.Config } } @@ -72,16 +78,31 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return nil, err } + if b.Config.TLS.IsEnabled() { + tlsConfig, err := tlscommon.LoadTLSConfig(b.Config.TLS) + if err != nil { + return nil, fmt.Errorf("could not load provided TLS configuration: %w", err) + } + + b.Config.TLSConfig = tlsConfig.ToConfig() + } + return b, nil } // Fetch fetches status messages from a mysql host. func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) error { if m.db == nil { + if m.Config.TLSConfig != nil { + err := mysqlDriver.RegisterTLSConfig(mysql.TLSConfigKey, m.Config.TLSConfig) + if err != nil { + return fmt.Errorf("registering custom tls config failed: %w", err) + } + } var err error m.db, err = sql.NewDBClient("mysql", m.HostData().URI, m.Logger()) if err != nil { - return fmt.Errorf("mysql-status fetch failed: %w", err) + return fmt.Errorf("mysql-query fetch failed: %w", err) } } diff --git a/metricbeat/module/mysql/status/status.go b/metricbeat/module/mysql/status/status.go index dd57f7e23c92..ac3e5b83a185 100644 --- a/metricbeat/module/mysql/status/status.go +++ b/metricbeat/module/mysql/status/status.go @@ -40,20 +40,25 @@ func init() { // MetricSet for fetching MySQL server status. type MetricSet struct { - mb.BaseMetricSet + *mysql.Metricset db *sql.DB } // New creates and returns a new MetricSet instance. func New(base mb.BaseMetricSet) (mb.MetricSet, error) { - return &MetricSet{BaseMetricSet: base}, nil + ms, err := mysql.NewMetricset(base) + if err != nil { + return nil, err + } + + return &MetricSet{Metricset: ms, db: nil}, nil } // Fetch fetches status messages from a mysql host. func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { if m.db == nil { var err error - m.db, err = mysql.NewDB(m.HostData().URI) + m.db, err = mysql.NewDB(m.HostData().URI, m.Metricset.Config.TLSConfig) if err != nil { return fmt.Errorf("mysql-status fetch failed: %w", err) } diff --git a/metricbeat/modules.d/mysql.yml.disabled b/metricbeat/modules.d/mysql.yml.disabled index 2913f5af8bc8..27dcc1e59ea5 100644 --- a/metricbeat/modules.d/mysql.yml.disabled +++ b/metricbeat/modules.d/mysql.yml.disabled @@ -21,3 +21,15 @@ # Password of hosts. Empty by default. #password: secret + + # Optional SSL/TLS. By default is false. + #ssl.enabled: true + + # List of root certificates for SSL/TLS server verification + #ssl.certificate_authorities: ["/etc/pki/root/ca.crt"] + + # Certificate for SSL/TLS client authentication + #ssl.certificate: "/etc/pki/client/cert.crt" + + # Client certificate key file + #ssl.key: "/etc/pki/client/cert.key" \ No newline at end of file diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index 3e0cd8a6d90b..5cae69cbdbe3 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.14.0-bd8c2be5-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.14.0-09a11c1c-SNAPSHOT # When extend is used it merges healthcheck.tests, see: # https://github.com/docker/compose/issues/8962 # healthcheck: @@ -31,7 +31,7 @@ services: - "./docker/elasticsearch/users_roles:/usr/share/elasticsearch/config/users_roles" logstash: - image: docker.elastic.co/logstash/logstash:8.14.0-bd8c2be5-SNAPSHOT + image: docker.elastic.co/logstash/logstash:8.14.0-09a11c1c-SNAPSHOT healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9600/_node/stats"] retries: 600 @@ -44,7 +44,7 @@ services: - 5055:5055 kibana: - image: docker.elastic.co/kibana/kibana:8.14.0-bd8c2be5-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.14.0-09a11c1c-SNAPSHOT environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing" diff --git a/x-pack/filebeat/input/cel/config_auth.go b/x-pack/filebeat/input/cel/config_auth.go index d6b35d633e69..ac187f4ffa1e 100644 --- a/x-pack/filebeat/input/cel/config_auth.go +++ b/x-pack/filebeat/input/cel/config_auth.go @@ -6,7 +6,6 @@ package cel import ( "context" - "crypto/x509" "encoding/json" "errors" "fmt" @@ -341,7 +340,10 @@ func (o *oAuth2Config) validateOktaProvider() error { } // jwk_pem if o.OktaJWKPEM != "" { - _, err := x509.ParsePKCS1PrivateKey([]byte(o.OktaJWKPEM)) + _, err := pemPKCS8PrivateKey([]byte(o.OktaJWKPEM)) + if err != nil { + return fmt.Errorf("okta validation error: %w", err) + } return err } // jwk_file diff --git a/x-pack/filebeat/input/cel/config_okta_auth.go b/x-pack/filebeat/input/cel/config_okta_auth.go index 74366afd3d5f..0f18b12e66ce 100644 --- a/x-pack/filebeat/input/cel/config_okta_auth.go +++ b/x-pack/filebeat/input/cel/config_okta_auth.go @@ -12,6 +12,7 @@ import ( "encoding/base64" "encoding/json" "encoding/pem" + "errors" "fmt" "math/big" "net/http" @@ -160,17 +161,24 @@ func (i *base64int) UnmarshalJSON(b []byte) error { } func generateOktaJWTPEM(pemdata string, cnf *oauth2.Config) (string, error) { - blk, rest := pem.Decode([]byte(pemdata)) - if rest := bytes.TrimSpace(rest); len(rest) != 0 { - return "", fmt.Errorf("PEM text has trailing data: %s", rest) - } - key, err := x509.ParsePKCS8PrivateKey(blk.Bytes) + key, err := pemPKCS8PrivateKey([]byte(pemdata)) if err != nil { return "", err } return signJWT(cnf, key) } +func pemPKCS8PrivateKey(pemdata []byte) (any, error) { + blk, rest := pem.Decode(pemdata) + if rest := bytes.TrimSpace(rest); len(rest) != 0 { + return nil, fmt.Errorf("PEM text has trailing data: %d bytes", len(rest)) + } + if blk == nil { + return nil, errors.New("no PEM data") + } + return x509.ParsePKCS8PrivateKey(blk.Bytes) +} + // signJWT creates a JWT token using required claims and sign it with the // private key. func signJWT(cnf *oauth2.Config, key any) (string, error) { @@ -182,7 +190,7 @@ func signJWT(cnf *oauth2.Config, key any) (string, error) { Expiration(now.Add(time.Hour)). Build() if err != nil { - return "", err + return "", fmt.Errorf("failed to create token: %w", err) } signedToken, err := jwt.Sign(tok, jwt.WithKey(jwa.RS256, key)) if err != nil { diff --git a/x-pack/filebeat/input/cel/config_test.go b/x-pack/filebeat/input/cel/config_test.go index 7acf74df08ca..0a686df099c1 100644 --- a/x-pack/filebeat/input/cel/config_test.go +++ b/x-pack/filebeat/input/cel/config_test.go @@ -539,6 +539,47 @@ var oAuth2ValidationTests = []struct { }, }, }, + { + name: "okta_successful_pem_oauth2_validation", + input: map[string]interface{}{ + "auth.oauth2": map[string]interface{}{ + "provider": "okta", + "client.id": "a_client_id", + "token_url": "localhost", + "scopes": []string{"foo"}, + "okta.jwk_pem": ` +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCOuef3HMRhohVT +5kSoAJgV+atpDjkwTwkOq+ImnbBlv75GaApG90w8VpjXjhqN/1KJmwfyrKiquiMq +OPu+o/672Dys5rUAaWSbT7wRF1GjLDDZrM0GHRdV4DGxM/LKI8I5yE1Mx3EzV+D5 +ZLmcRc5U4oEoMwtGpr0zRZ7uUr6a28UQwcUsVIPItc1/9rERlo1WTv8dcaj4ECC3 +2Sc0y/F+9XqwJvLd4Uv6ckzP0Sv4tbDA+7jpD9MneAIUiZ4LVj2cwbBd+YRY6jXx +MkevcCSmSX60clBY1cIFkw1DYHqtdHEwAQcQHLGMoi72xRP2qrdzIPsaTKVYoHVo +WA9vADdHAgMBAAECggEAIlx7jjCsztyYyeQsL05FTzUWoWo9NnYwtgmHnshkCXsK +MiUmJEOxZO1sSqj5l6oakupyFWigCspZYPbrFNCiqVK7+NxqQzkccY/WtT6p9uDS +ufUyPwCN96zMCd952lSVlBe3FH8Hr9a+YQxw60CbFjCZ67WuR0opTsi6JKJjJSDb +TQQZ4qJR97D05I1TgfmO+VO7G/0/dDaNHnnlYz0AnOgZPSyvrU2G5cYye4842EMB +ng81xjHD+xp55JNui/xYkhmYspYhrB2KlEjkKb08OInUjBeaLEAgA1r9yOHsfV/3 +DQzDPRO9iuqx5BfJhdIqUB1aifrye+sbxt9uMBtUgQKBgQDVdfO3GYT+ZycOQG9P +QtdMn6uiSddchVCGFpk331u6M6yafCKjI/MlJDl29B+8R5sVsttwo8/qnV/xd3cn +pY14HpKAsE4l6/Ciagzoj+0NqfPEDhEzbo8CyArcd7pSxt3XxECAfZe2+xivEPHe +gFO60vSFjFtvlLRMDMOmqX3kYQKBgQCrK1DISyQTnD6/axsgh2/ESOmT7n+JRMx/ +YzA7Lxu3zGzUC8/sRDa1C41t054nf5ZXJueYLDSc4kEAPddzISuCLxFiTD2FQ75P +lHWMgsEzQObDm4GPE9cdKOjoAvtAJwbvZcjDa029CDx7aCaDzbNvdmplZ7EUrznR +55U8Wsm8pwKBgBytxTmzZwfbCgdDJvFKNKzpwuCB9TpL+v6Y6Kr2Clfg+26iAPFU +MiWqUUInGGBuamqm5g6jI5sM28gQWeTsvC4IRXyes1Eq+uCHSQax15J/Y+3SSgNT +9kjUYYkvWMwoRcPobRYWSZze7XkP2L8hFJ7EGvAaZGqAWxzgliS9HtnhAoGAONZ/ +UqMw7Zoac/Ga5mhSwrj7ZvXxP6Gqzjofj+eKqrOlB5yMhIX6LJATfH6iq7cAMxxm +Fu/G4Ll4oB3o5wACtI3wldV/MDtYfJBtoCTjBqPsfNOsZ9hMvBATlsc2qwzKjsAb +tFhzTevoOYpSD75EcSS/G8Ec2iN9bagatBnpl00CgYBVqAOFZelNfP7dj//lpk8y +EUAw7ABOq0S9wkpFWTXIVPoBQUipm3iAUqGNPmvr/9ShdZC9xeu5AwKram4caMWJ +ExRhcDP1hFM6CdmSkIYEgBKvN9N0O4Lx1ba34gk74Hm65KXxokjJHOC0plO7c7ok +LNV/bIgMHOMoxiGrwyjAhg== +-----END PRIVATE KEY----- +`, + }, + }, + }, } func TestConfigOauth2Validation(t *testing.T) { diff --git a/x-pack/filebeat/input/httpjson/config_auth.go b/x-pack/filebeat/input/httpjson/config_auth.go index d05592dfa500..b25bab03dd39 100644 --- a/x-pack/filebeat/input/httpjson/config_auth.go +++ b/x-pack/filebeat/input/httpjson/config_auth.go @@ -6,7 +6,6 @@ package httpjson import ( "context" - "crypto/x509" "encoding/json" "errors" "fmt" @@ -309,8 +308,11 @@ func (o *oAuth2Config) validateOktaProvider() error { } // jwk_pem if o.OktaJWKPEM != "" { - _, err := x509.ParsePKCS1PrivateKey([]byte(o.OktaJWKPEM)) - return err + _, err := pemPKCS8PrivateKey([]byte(o.OktaJWKPEM)) + if err != nil { + return fmt.Errorf("okta validation error: %w", err) + } + return nil } // jwk_file if o.OktaJWKFile != "" { diff --git a/x-pack/filebeat/input/httpjson/config_okta_auth.go b/x-pack/filebeat/input/httpjson/config_okta_auth.go index c2b4289d9c91..8d2a8415c2e7 100644 --- a/x-pack/filebeat/input/httpjson/config_okta_auth.go +++ b/x-pack/filebeat/input/httpjson/config_okta_auth.go @@ -12,6 +12,7 @@ import ( "encoding/base64" "encoding/json" "encoding/pem" + "errors" "fmt" "math/big" "net/http" @@ -158,17 +159,24 @@ func (i *base64int) UnmarshalJSON(b []byte) error { } func generateOktaJWTPEM(pemdata string, cnf *oauth2.Config) (string, error) { - blk, rest := pem.Decode([]byte(pemdata)) - if rest := bytes.TrimSpace(rest); len(rest) != 0 { - return "", fmt.Errorf("PEM text has trailing data: %s", rest) - } - key, err := x509.ParsePKCS8PrivateKey(blk.Bytes) + key, err := pemPKCS8PrivateKey([]byte(pemdata)) if err != nil { return "", err } return signJWT(cnf, key) } +func pemPKCS8PrivateKey(pemdata []byte) (any, error) { + blk, rest := pem.Decode(pemdata) + if rest := bytes.TrimSpace(rest); len(rest) != 0 { + return nil, fmt.Errorf("PEM text has trailing data: %d bytes", len(rest)) + } + if blk == nil { + return nil, errors.New("no PEM data") + } + return x509.ParsePKCS8PrivateKey(blk.Bytes) +} + // signJWT creates a JWT token using required claims and sign it with the private key. func signJWT(cnf *oauth2.Config, key any) (string, error) { now := time.Now() @@ -179,7 +187,7 @@ func signJWT(cnf *oauth2.Config, key any) (string, error) { Expiration(now.Add(time.Hour)). Build() if err != nil { - return "", err + return "", fmt.Errorf("failed to create token: %w", err) } signedToken, err := jwt.Sign(tok, jwt.WithKey(jwa.RS256, key)) if err != nil { diff --git a/x-pack/filebeat/input/httpjson/config_test.go b/x-pack/filebeat/input/httpjson/config_test.go index d88c6ac4a625..910510b6e9cc 100644 --- a/x-pack/filebeat/input/httpjson/config_test.go +++ b/x-pack/filebeat/input/httpjson/config_test.go @@ -499,6 +499,47 @@ func TestConfigOauth2Validation(t *testing.T) { }, }, }, + { + name: "okta successful pem oauth2 validation", + input: map[string]interface{}{ + "auth.oauth2": map[string]interface{}{ + "provider": "okta", + "client.id": "a_client_id", + "token_url": "localhost", + "scopes": []string{"foo"}, + "okta.jwk_pem": ` +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCOuef3HMRhohVT +5kSoAJgV+atpDjkwTwkOq+ImnbBlv75GaApG90w8VpjXjhqN/1KJmwfyrKiquiMq +OPu+o/672Dys5rUAaWSbT7wRF1GjLDDZrM0GHRdV4DGxM/LKI8I5yE1Mx3EzV+D5 +ZLmcRc5U4oEoMwtGpr0zRZ7uUr6a28UQwcUsVIPItc1/9rERlo1WTv8dcaj4ECC3 +2Sc0y/F+9XqwJvLd4Uv6ckzP0Sv4tbDA+7jpD9MneAIUiZ4LVj2cwbBd+YRY6jXx +MkevcCSmSX60clBY1cIFkw1DYHqtdHEwAQcQHLGMoi72xRP2qrdzIPsaTKVYoHVo +WA9vADdHAgMBAAECggEAIlx7jjCsztyYyeQsL05FTzUWoWo9NnYwtgmHnshkCXsK +MiUmJEOxZO1sSqj5l6oakupyFWigCspZYPbrFNCiqVK7+NxqQzkccY/WtT6p9uDS +ufUyPwCN96zMCd952lSVlBe3FH8Hr9a+YQxw60CbFjCZ67WuR0opTsi6JKJjJSDb +TQQZ4qJR97D05I1TgfmO+VO7G/0/dDaNHnnlYz0AnOgZPSyvrU2G5cYye4842EMB +ng81xjHD+xp55JNui/xYkhmYspYhrB2KlEjkKb08OInUjBeaLEAgA1r9yOHsfV/3 +DQzDPRO9iuqx5BfJhdIqUB1aifrye+sbxt9uMBtUgQKBgQDVdfO3GYT+ZycOQG9P +QtdMn6uiSddchVCGFpk331u6M6yafCKjI/MlJDl29B+8R5sVsttwo8/qnV/xd3cn +pY14HpKAsE4l6/Ciagzoj+0NqfPEDhEzbo8CyArcd7pSxt3XxECAfZe2+xivEPHe +gFO60vSFjFtvlLRMDMOmqX3kYQKBgQCrK1DISyQTnD6/axsgh2/ESOmT7n+JRMx/ +YzA7Lxu3zGzUC8/sRDa1C41t054nf5ZXJueYLDSc4kEAPddzISuCLxFiTD2FQ75P +lHWMgsEzQObDm4GPE9cdKOjoAvtAJwbvZcjDa029CDx7aCaDzbNvdmplZ7EUrznR +55U8Wsm8pwKBgBytxTmzZwfbCgdDJvFKNKzpwuCB9TpL+v6Y6Kr2Clfg+26iAPFU +MiWqUUInGGBuamqm5g6jI5sM28gQWeTsvC4IRXyes1Eq+uCHSQax15J/Y+3SSgNT +9kjUYYkvWMwoRcPobRYWSZze7XkP2L8hFJ7EGvAaZGqAWxzgliS9HtnhAoGAONZ/ +UqMw7Zoac/Ga5mhSwrj7ZvXxP6Gqzjofj+eKqrOlB5yMhIX6LJATfH6iq7cAMxxm +Fu/G4Ll4oB3o5wACtI3wldV/MDtYfJBtoCTjBqPsfNOsZ9hMvBATlsc2qwzKjsAb +tFhzTevoOYpSD75EcSS/G8Ec2iN9bagatBnpl00CgYBVqAOFZelNfP7dj//lpk8y +EUAw7ABOq0S9wkpFWTXIVPoBQUipm3iAUqGNPmvr/9ShdZC9xeu5AwKram4caMWJ +ExRhcDP1hFM6CdmSkIYEgBKvN9N0O4Lx1ba34gk74Hm65KXxokjJHOC0plO7c7ok +LNV/bIgMHOMoxiGrwyjAhg== +-----END PRIVATE KEY----- +`, + }, + }, + }, } for _, c := range cases { diff --git a/x-pack/filebeat/input/httpjson/encoding.go b/x-pack/filebeat/input/httpjson/encoding.go index 5dd62f105351..a7da4f25c0dc 100644 --- a/x-pack/filebeat/input/httpjson/encoding.go +++ b/x-pack/filebeat/input/httpjson/encoding.go @@ -16,6 +16,7 @@ import ( "net/http" "unicode" + "github.com/elastic/elastic-agent-libs/mapstr" "github.com/elastic/mito/lib/xml" ) @@ -64,13 +65,14 @@ type decoderFunc func(p []byte, dst *response) error // encodeAsJSON encodes trReq as a JSON message. func encodeAsJSON(trReq transformable) ([]byte, error) { - if len(trReq.body()) == 0 { + body, err := trReq.GetValue("body") + if err == mapstr.ErrKeyNotFound { return nil, nil } header := trReq.header() header.Set("Content-Type", "application/json") trReq.setHeader(header) - return json.Marshal(trReq.body()) + return json.Marshal(body) } // decodeAsJSON decodes the JSON message in p into dst. diff --git a/x-pack/filebeat/input/httpjson/input_test.go b/x-pack/filebeat/input/httpjson/input_test.go index 498ccc861834..6604c999d447 100644 --- a/x-pack/filebeat/input/httpjson/input_test.go +++ b/x-pack/filebeat/input/httpjson/input_test.go @@ -93,6 +93,17 @@ var testCases = []struct { handler: defaultHandler(http.MethodPost, `{"test":"abc"}`, ""), expected: []string{`{"hello":[{"world":"moon"},{"space":[{"cake":"pumpkin"}]}]}`}, }, + { + name: "POST_request_with_empty_object_body", + setupServer: newTestServer(httptest.NewServer), + baseConfig: map[string]interface{}{ + "interval": 1, + "request.method": http.MethodPost, + "request.body": map[string]interface{}{}, + }, + handler: defaultHandler(http.MethodPost, `{}`, ""), + expected: []string{`{"hello":[{"world":"moon"},{"space":[{"cake":"pumpkin"}]}]}`}, + }, { name: "repeated_POST_requests", setupServer: newTestServer(httptest.NewServer), @@ -1516,7 +1527,7 @@ func defaultHandler(expectedMethod, expectedBody, msg string) http.HandlerFunc { r.Body.Close() if expectedBody != string(body) { w.WriteHeader(http.StatusBadRequest) - msg = fmt.Sprintf(`{"error":"expected body was %q"}`, expectedBody) + msg = fmt.Sprintf(`{"error":"expected body was %q, but got %q"}`, expectedBody, body) } } diff --git a/x-pack/filebeat/input/httpjson/request.go b/x-pack/filebeat/input/httpjson/request.go index 9e60d22ac495..3e63f0267162 100644 --- a/x-pack/filebeat/input/httpjson/request.go +++ b/x-pack/filebeat/input/httpjson/request.go @@ -440,7 +440,7 @@ func (rf *requestFactory) newRequest(ctx *transformContext) (transformable, erro req := transformable{} req.setURL(rf.url) - if rf.body != nil && len(*rf.body) > 0 { + if rf.body != nil { req.setBody(rf.body.Clone()) } diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 1e6abf11a604..f71e58904fd7 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -1172,6 +1172,18 @@ metricbeat.modules: # By setting raw to true, all raw fields from the status metricset will be added to the event. #raw: false + # Optional SSL/TLS. By default is false. + #ssl.enabled: true + + # List of root certificates for SSL/TLS server verification + #ssl.certificate_authorities: ["/etc/pki/root/ca.crt"] + + # Certificate for SSL/TLS client authentication + #ssl.certificate: "/etc/pki/client/cert.crt" + + # Client certificate key file + #ssl.key: "/etc/pki/client/cert.key" + #--------------------------------- NATS Module --------------------------------- - module: nats metricsets: