From aff596a5bd3c20d49fd223f5ab53628026e833b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Constan=C3=A7a=20Manteigas?= <113898685+constanca-m@users.noreply.github.com> Date: Tue, 20 Feb 2024 13:58:41 +0100 Subject: [PATCH 01/26] Update control plane toleration. (#38040) Signed-off-by: constanca --- auditbeat/docs/running-on-kubernetes.asciidoc | 8 ++++---- filebeat/docs/running-on-kubernetes.asciidoc | 12 ++++++------ metricbeat/docs/running-on-kubernetes.asciidoc | 10 +++++----- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/auditbeat/docs/running-on-kubernetes.asciidoc b/auditbeat/docs/running-on-kubernetes.asciidoc index 73ac5cdd70f4..f5f4f0f4715e 100644 --- a/auditbeat/docs/running-on-kubernetes.asciidoc +++ b/auditbeat/docs/running-on-kubernetes.asciidoc @@ -57,17 +57,17 @@ may want to change that behavior, so just edit the YAML file and modify them: ------------------------------------------------ [float] -===== Running {beatname_uc} on master nodes +===== Running {beatname_uc} on control plane nodes -Kubernetes master nodes can use https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/[taints] -to limit the workloads that can run on them. To run {beatname_uc} on master nodes you may need to +Kubernetes control plane nodes can use https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/[taints] +to limit the workloads that can run on them. To run {beatname_uc} on control plane nodes you may need to update the Daemonset spec to include proper tolerations: [source,yaml] ------------------------------------------------ spec: tolerations: - - key: node-role.kubernetes.io/master + - key: node-role.kubernetes.io/control-plane effect: NoSchedule ------------------------------------------------ diff --git a/filebeat/docs/running-on-kubernetes.asciidoc b/filebeat/docs/running-on-kubernetes.asciidoc index acc2905ae065..889ec8d5d8b7 100644 --- a/filebeat/docs/running-on-kubernetes.asciidoc +++ b/filebeat/docs/running-on-kubernetes.asciidoc @@ -61,17 +61,17 @@ in the manifest file: ------------------------------------------------ [float] -===== Running {beatname_uc} on master nodes +===== Running {beatname_uc} on control plane nodes -Kubernetes master nodes can use https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/[taints] -to limit the workloads that can run on them. To run {beatname_uc} on master nodes you may need to +Kubernetes control plane nodes can use https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/[taints] +to limit the workloads that can run on them. To run {beatname_uc} on control plane nodes you may need to update the Daemonset spec to include proper tolerations: [source,yaml] ------------------------------------------------ spec: tolerations: - - key: node-role.kubernetes.io/master + - key: node-role.kubernetes.io/control-plane effect: NoSchedule ------------------------------------------------ @@ -110,7 +110,7 @@ oc patch namespace kube-system -p \ ---- + This command sets the node selector for the project to an empty string. If you -don't run this command, the default node selector will skip master nodes. +don't run this command, the default node selector will skip control plane nodes. In order to support runtime environments with Openshift (eg. CRI-O, containerd) you need to configure following path: @@ -137,7 +137,7 @@ filebeat.autodiscover: - /var/log/containers/*.log ---- -NOTE: `/var/log/containers/\*.log` is normally a symlink to `/var/log/pods/*/*.log`, +NOTE: `/var/log/containers/\*.log` is normally a symlink to `/var/log/pods/*/*.log`, so above paths can be edited accordingly diff --git a/metricbeat/docs/running-on-kubernetes.asciidoc b/metricbeat/docs/running-on-kubernetes.asciidoc index 2fcad7f2cf1d..e2ac0be96646 100644 --- a/metricbeat/docs/running-on-kubernetes.asciidoc +++ b/metricbeat/docs/running-on-kubernetes.asciidoc @@ -70,17 +70,17 @@ in the manifest file: ------------------------------------------------ [float] -===== Running {beatname_uc} on master nodes +===== Running {beatname_uc} on control plane nodes -Kubernetes master nodes can use https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/[taints] -to limit the workloads that can run on them. To run {beatname_uc} on master nodes you may need to +Kubernetes control plane nodes can use https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/[taints] +to limit the workloads that can run on them. To run {beatname_uc} on control plane nodes you may need to update the Daemonset spec to include proper tolerations: [source,yaml] ------------------------------------------------ spec: tolerations: - - key: node-role.kubernetes.io/master + - key: node-role.kubernetes.io/control-plane effect: NoSchedule ------------------------------------------------ @@ -166,7 +166,7 @@ oc patch namespace kube-system -p \ ---- + This command sets the node selector for the project to an empty string. If you -don't run this command, the default node selector will skip master nodes. +don't run this command, the default node selector will skip control plane nodes. NOTE: for openshift versions prior to the version 4.x additionally you need to modify the `DaemonSet` container spec in the manifest file to enable the container to run as privileged: [source,yaml] From 865f70f8bf50a0214bad2a94a806df4bbdb36a3a Mon Sep 17 00:00:00 2001 From: David Kilfoyle <41695641+kilfoyle@users.noreply.github.com> Date: Tue, 20 Feb 2024 10:21:32 -0500 Subject: [PATCH 02/26] Add LVM example to Filebeat file / device ID docs (#37959) * Add LVM example to Filebeat file / device ID docs * Move addition to under 'native' in the file_identity section --- filebeat/docs/inputs/input-filestream-file-options.asciidoc | 3 +++ filebeat/docs/inputs/input-filestream.asciidoc | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/filebeat/docs/inputs/input-filestream-file-options.asciidoc b/filebeat/docs/inputs/input-filestream-file-options.asciidoc index db00a8fe7662..a3be665e28e9 100644 --- a/filebeat/docs/inputs/input-filestream-file-options.asciidoc +++ b/filebeat/docs/inputs/input-filestream-file-options.asciidoc @@ -527,6 +527,9 @@ duplicated events in the output. *`native`*:: The default behaviour of {beatname_uc} is to differentiate between files using their inodes and device ids. ++ +In some cases these values can change during the lifetime of a file. +For example, when using the Linux link:https://en.wikipedia.org/wiki/Logical_Volume_Manager_%28Linux%29[LVM] (Logical Volume Manager), device numbers are allocated dynamically at module load (refer to link:https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/7/html/logical_volume_manager_administration/lv#persistent_numbers[Persistent Device Numbers] in the Red Hat Enterprise Linux documentation). To avoid the possibility of data duplication in this case, you can set `file_identity` to `path` rather than `native`. [source,yaml] ---- diff --git a/filebeat/docs/inputs/input-filestream.asciidoc b/filebeat/docs/inputs/input-filestream.asciidoc index e55ff6114962..47d1b24a8e85 100644 --- a/filebeat/docs/inputs/input-filestream.asciidoc +++ b/filebeat/docs/inputs/input-filestream.asciidoc @@ -94,7 +94,7 @@ By default, {beatname_uc} identifies files based on their inodes and device IDs. However, on network shares and cloud providers these values might change during the lifetime of the file. If this happens {beatname_uc} thinks that file is new and resends the whole content -of the file. To solve this problem you can configure `file_identity` option. Possible +of the file. To solve this problem you can configure the `file_identity` option. Possible values besides the default `inode_deviceid` are `path`, `inode_marker` and `fingerprint`. WARNING: Changing `file_identity` methods between runs may result in From 1f38627297cadfd3f56b00623da0278c8e39a58e Mon Sep 17 00:00:00 2001 From: sharbuz <87968844+sharbuz@users.noreply.github.com> Date: Tue, 20 Feb 2024 19:09:40 +0200 Subject: [PATCH 03/26] Migrate xpack libbeat pipeline (#38006) Migrate x-pack libbeat pipeline Fix the Windows script remove the Choco --------- Co-authored-by: Victor Martinez --- .buildkite/hooks/pre-command | 2 +- .buildkite/scripts/common.sh | 84 +++++++++--- .../generate_xpack_libbeat_pipeline.sh | 127 ++++++++++++++++++ .buildkite/scripts/setenv.sh | 3 +- .buildkite/scripts/win_unit_tests.ps1 | 127 ++++++++++++++---- .buildkite/x-pack/pipeline.xpack.libbeat.yml | 49 ++++++- 6 files changed, 348 insertions(+), 44 deletions(-) create mode 100755 .buildkite/scripts/generate_xpack_libbeat_pipeline.sh diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 4dda7a884dda..0ac7c51099c0 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -12,7 +12,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == fi fi -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-winlogbeat" ]]; then +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-winlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-libbeat" ]]; then source .buildkite/scripts/setenv.sh if [[ "${BUILDKITE_COMMAND}" =~ ^buildkite-agent ]]; then echo "Skipped pre-command when running the Upload pipeline" diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 6e7e19ea7d2d..b797ec718aa7 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -9,7 +9,6 @@ arch_type="$(uname -m)" GITHUB_PR_TRIGGER_COMMENT=${GITHUB_PR_TRIGGER_COMMENT:-""} GITHUB_PR_LABELS=${GITHUB_PR_LABELS:-""} ONLY_DOCS=${ONLY_DOCS:-"true"} - [ -z "${run_libbeat+x}" ] && run_libbeat="$(buildkite-agent meta-data get run_libbeat --default "false")" [ -z "${run_metricbeat+x}" ] && run_metricbeat="$(buildkite-agent meta-data get run_metricbeat --default "false")" [ -z "${run_packetbeat+x}" ] && run_packetbeat="$(buildkite-agent meta-data get run_packetbeat --default "false")" @@ -18,9 +17,6 @@ ONLY_DOCS=${ONLY_DOCS:-"true"} [ -z "${run_packetbeat_arm_tests+x}" ] && run_packetbeat_arm_tests="$(buildkite-agent meta-data get run_packetbeat_arm_tests --default "false")" [ -z "${run_metricbeat_macos_tests+x}" ] && run_metricbeat_macos_tests="$(buildkite-agent meta-data get run_metricbeat_macos_tests --default "false")" [ -z "${run_packetbeat_macos_tests+x}" ] && run_packetbeat_macos_tests="$(buildkite-agent meta-data get run_packetbeat_macos_tests --default "false")" -trigger_specific_beat="run_${BEATS_PROJECT_NAME}" -trigger_specific_arm_tests="run_${BEATS_PROJECT_NAME}_arm_tests" -trigger_specific_macos_tests="run_${BEATS_PROJECT_NAME}_macos_tests" metricbeat_changeset=( "^metricbeat/.*" @@ -38,6 +34,30 @@ winlogbeat_changeset=( "^winlogbeat/.*" ) +xpack_libbeat_changeset=( + "^x-pack/libbeat/.*" + ) + +xpack_metricbeat_changeset=( + "^x-pack/metricbeat/.*" + ) + +xpack_packetbeat_changeset=( + "^x-pack/packetbeat/.*" + ) + +xpack_winlogbeat_changeset=( + "^x-pack/winlogbeat/.*" + ) + +ci_changeset=( + "^.buildkite/.*" + ) + +go_mod_changeset=( + "^go.mod" + ) + oss_changeset=( "^go.mod" "^pytest.ini" @@ -46,14 +66,11 @@ oss_changeset=( "^testing/.*" ) -ci_changeset=( - "^.buildkite/.*" +xpack_changeset=( + "${xpack_libbeat_changeset[@]}" + "${oss_changeset[@]}" ) -go_mod_changeset=( - "^go.mod" - ) - docs_changeset=( ".*\\.(asciidoc|md)" "deploy/kubernetes/.*-kubernetes\\.yaml" @@ -64,6 +81,41 @@ packaging_changeset=( ".go-version" ) +check_and_set_beat_vars() { + if [[ -n "$BEATS_PROJECT_NAME" && "$BEATS_PROJECT_NAME" == *"x-pack/"* ]]; then + BEATS_XPACK_PROJECT_NAME=${BEATS_PROJECT_NAME//-/} #remove - + BEATS_XPACK_PROJECT_NAME=${BEATS_XPACK_PROJECT_NAME//\//_} #replace / to _ + BEATS_XPACK_LABEL_PROJECT_NAME=${BEATS_PROJECT_NAME//\//-} #replace / to - for labels + BEATS_GH_LABEL=${BEATS_XPACK_LABEL_PROJECT_NAME} + TRIGGER_SPECIFIC_BEAT="run_${BEATS_XPACK_PROJECT_NAME}" + TRIGGER_SPECIFIC_ARM_TESTS="run_${BEATS_XPACK_PROJECT_NAME}_arm_tests" + TRIGGER_SPECIFIC_MACOS_TESTS="run_${BEATS_XPACK_PROJECT_NAME}_macos_tests" + declare -n BEAT_CHANGESET_REFERENCE="${BEATS_XPACK_PROJECT_NAME}_changeset" + echo "Beats project name is $BEATS_XPACK_PROJECT_NAME" + mandatory_changeset=( + "${BEAT_CHANGESET_REFERENCE[@]}" + "${xpack_changeset[@]}" + "${ci_changeset[@]}" + ) + else + BEATS_GH_LABEL=${BEATS_PROJECT_NAME} + TRIGGER_SPECIFIC_BEAT="run_${BEATS_PROJECT_NAME}" + TRIGGER_SPECIFIC_ARM_TESTS="run_${BEATS_PROJECT_NAME}_arm_tests" + TRIGGER_SPECIFIC_MACOS_TESTS="run_${BEATS_PROJECT_NAME}_macos_tests" + declare -n BEAT_CHANGESET_REFERENCE="${BEATS_PROJECT_NAME}_changeset" + echo "Beats project name is $BEATS_PROJECT_NAME" + mandatory_changeset=( + "${BEAT_CHANGESET_REFERENCE[@]}" + "${oss_changeset[@]}" + "${ci_changeset[@]}" + ) + fi + BEATS_GH_COMMENT="/test ${BEATS_PROJECT_NAME}" + BEATS_GH_MACOS_COMMENT="${BEATS_GH_COMMENT} for macos" + BEATS_GH_ARM_COMMENT="${BEATS_GH_COMMENT} for arm" + BAETS_GH_MACOS_LABEL="macOS" + BAETS_GH_ARM_LABEL="arm" +} with_docker_compose() { local version=$1 @@ -240,11 +292,7 @@ are_changed_only_paths() { } are_conditions_met_mandatory_tests() { - declare -n beat_changeset_reference="${BEATS_PROJECT_NAME}_changeset" - if are_paths_changed "${oss_changeset[@]}" || are_paths_changed "${ci_changeset[@]}" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 - return 0 - fi - if are_paths_changed "${beat_changeset_reference[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test ${BEATS_PROJECT_NAME}" || "${GITHUB_PR_LABELS}" =~ /(?i)${BEATS_PROJECT_NAME}/ || "${!trigger_specific_beat}" == "true" ]]; then + if are_paths_changed "${mandatory_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_COMMENT}" || "${GITHUB_PR_LABELS}" =~ /(?i)${BEATS_GH_LABEL}/ || "${!TRIGGER_SPECIFIC_BEAT}" == "true" ]]; then return 0 fi return 1 @@ -253,7 +301,7 @@ are_conditions_met_mandatory_tests() { are_conditions_met_arm_tests() { if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then - if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test ${BEATS_PROJECT_NAME} for arm" || "${GITHUB_PR_LABELS}" =~ arm || "${!trigger_specific_arm_tests}" == "true" ]]; then + if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_ARM_COMMENT}" || "${GITHUB_PR_LABELS}" =~ "${BAETS_GH_ARM_LABEL}" || "${!TRIGGER_SPECIFIC_ARM_TESTS}" == "true" ]]; then return 0 fi fi @@ -264,7 +312,7 @@ are_conditions_met_arm_tests() { are_conditions_met_macos_tests() { if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then - if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test ${BEATS_PROJECT_NAME} for macos" || "${GITHUB_PR_LABELS}" =~ macOS || "${!trigger_specific_macos_tests}" == "true" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 + if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_MACOS_COMMENT}" || "${GITHUB_PR_LABELS}" =~ "${BAETS_GH_MACOS_LABEL}" || "${!TRIGGER_SPECIFIC_MACOS_TESTS}" == "true" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 return 0 fi fi @@ -302,3 +350,5 @@ fi if are_paths_changed "${packaging_changeset[@]}" ; then PACKAGING_CHANGES="true" fi + +check_and_set_beat_vars diff --git a/.buildkite/scripts/generate_xpack_libbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_libbeat_pipeline.sh new file mode 100755 index 000000000000..66f0750ab6fa --- /dev/null +++ b/.buildkite/scripts/generate_xpack_libbeat_pipeline.sh @@ -0,0 +1,127 @@ +#!/usr/bin/env bash + +source .buildkite/scripts/common.sh + +set -euo pipefail + +pipelineName="pipeline.libbeat-dynamic.yml" + +echo "Add the mandatory and extended tests without additional conditions into the pipeline" +if are_conditions_met_mandatory_tests; then + cat > $pipelineName <<- YAML + +steps: + + - group: "Mandatory Tests" + key: "mandatory-tests" + steps: + - label: ":linux: Ubuntu Unit Tests" + key: "mandatory-linux-unit-test" + command: ".buildkite/scripts/unit_tests.sh" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + + - label: ":go: Go Integration Tests" + key: "mandatory-int-test" + command: ".buildkite/scripts/go_int_tests.sh" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + + - label: ":python: Python Integration Tests" + key: "mandatory-python-int-test" + command: ".buildkite/scripts/py_int_tests.sh" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + + - label: ":windows: Windows Unit Tests - {{matrix.image}}" + command: ".buildkite/scripts/win_unit_tests.ps1" + key: "mandatory-win-unit-tests" + agents: + provider: "gcp" + image: "{{matrix.image}}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + matrix: + setup: + image: + - "${IMAGE_WIN_2016}" + - "${IMAGE_WIN_2022}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +### TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h + - group: "Extended Windows Tests" + key: "extended-win-tests" + steps: + - label: ":windows: Win 2019 Unit Tests" + key: "extended-win-2019-unit-tests" + command: ".buildkite/scripts/win_unit_tests.ps1" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2019}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":windows: Windows 10 Unit Tests" + key: "extended-win-10-unit-tests" + command: ".buildkite/scripts/win_unit_tests.ps1" + agents: + provider: "gcp" + image: "${IMAGE_WIN_10}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":windows: Windows 11 Unit Tests" + key: "extended-win-11-unit-tests" + command: ".buildkite/scripts/win_unit_tests.ps1" + agents: + provider: "gcp" + image: "${IMAGE_WIN_11}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +YAML +else + echo "The conditions don't match to requirements for generating pipeline steps." + exit 0 +fi + +echo "Check and add the Extended Tests into the pipeline" +if are_conditions_met_arm_tests; then + cat >> $pipelineName <<- YAML + + - group: "Extended Tests" + key: "extended-tests" + steps: + - label: ":linux: Arm64 Unit Tests" + key: "extended-arm64-unit-tests" + command: ".buildkite/scripts/unit_tests.sh" + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + +YAML +fi + +echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public +cat $pipelineName + +echo "--- Loading dynamic steps" +buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh index dbf23c3e5ef9..25121de212fd 100755 --- a/.buildkite/scripts/setenv.sh +++ b/.buildkite/scripts/setenv.sh @@ -6,13 +6,14 @@ SETUP_GVM_VERSION="v0.5.1" DOCKER_COMPOSE_VERSION="1.21.0" DOCKER_COMPOSE_VERSION_AARCH64="v2.21.0" SETUP_WIN_PYTHON_VERSION="3.11.0" +NMAP_WIN_VERSION="7.12" # Earlier versions of NMap provide WinPcap (the winpcap packages don't install nicely because they pop-up a UI) GO_VERSION=$(cat .go-version) - export SETUP_GVM_VERSION export DOCKER_COMPOSE_VERSION export DOCKER_COMPOSE_VERSION_AARCH64 export SETUP_WIN_PYTHON_VERSION +export NMAP_WIN_VERSION export GO_VERSION exportVars() { diff --git a/.buildkite/scripts/win_unit_tests.ps1 b/.buildkite/scripts/win_unit_tests.ps1 index da0ffc105a51..b3c5c58fac0a 100644 --- a/.buildkite/scripts/win_unit_tests.ps1 +++ b/.buildkite/scripts/win_unit_tests.ps1 @@ -1,5 +1,6 @@ $ErrorActionPreference = "Stop" # set -e -$WorkFolder = "metricbeat" +$WorkFolder = $env:BEATS_PROJECT_NAME +$WORKSPACE = Get-Location # Forcing to checkout again all the files with a correct autocrlf. # Doing this here because we cannot set git clone options before. function fixCRLF { @@ -8,30 +9,93 @@ function fixCRLF { git rm --quiet --cached -r . git reset --quiet --hard } -function withChoco { - Write-Host "-- Configure Choco --" - $env:ChocolateyInstall = Convert-Path "$((Get-Command choco).Path)\..\.." - Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1" + +function retry { + param( + [int]$retries, + [ScriptBlock]$scriptBlock + ) + $count = 0 + while ($count -lt $retries) { + $count++ + try { + & $scriptBlock + return + } catch { + $exitCode = $_.Exception.ErrorCode + Write-Host "Retry $count/$retries exited $exitCode, retrying..." + Start-Sleep -Seconds ([Math]::Pow(2, $count)) + } + } + Write-Host "Retry $count/$retries exited, no more retries left." +} + +function verifyFileChecksum { + param ( + [string]$filePath, + [string]$checksumFilePath + ) + $actualHash = (Get-FileHash -Algorithm SHA256 -Path $filePath).Hash + $checksumData = Get-Content -Path $checksumFilePath + $expectedHash = ($checksumData -split "\s+")[0] + if ($actualHash -eq $expectedHash) { + Write-Host "CheckSum is checked. File is correct. Original checkSum is: $expectedHash " + return $true + } else { + Write-Host "CheckSum is wrong. File can be corrupted or modified. Current checksum is: $actualHash, the original checksum is: $expectedHash" + return $false + } } + function withGolang($version) { - $downloadPath = Join-Path $env:TEMP "go_installer.msi" + Write-Host "-- Installing Go $version --" + $goDownloadPath = Join-Path $env:TEMP "go_installer.msi" $goInstallerUrl = "https://golang.org/dl/go$version.windows-amd64.msi" - Invoke-WebRequest -Uri $goInstallerUrl -OutFile $downloadPath - Start-Process -FilePath "msiexec.exe" -ArgumentList "/i $downloadPath /quiet" -Wait - $goBinPath = "${env:ProgramFiles}\Go\bin" - $env:Path += ";$goBinPath" + retry -retries 5 -scriptBlock { + Invoke-WebRequest -Uri $goInstallerUrl -OutFile $goDownloadPath + } + Start-Process -FilePath "msiexec.exe" -ArgumentList "/i $goDownloadPath /quiet" -Wait + $env:GOPATH = "${env:ProgramFiles}\Go" + $env:GOBIN = "${env:GOPATH}\bin" + $env:Path += ";$env:GOPATH;$env:GOBIN" go version + installGoDependencies } + function withPython($version) { - Write-Host "-- Install Python $version --" - choco install python --version=$version - refreshenv + Write-Host "-- Installing Python $version --" + [Net.ServicePointManager]::SecurityProtocol = "tls11, tls12, ssl3" + $pyDownloadPath = Join-Path $env:TEMP "python-$version-amd64.exe" + $pyInstallerUrl = "https://www.python.org/ftp/python/$version/python-$version-amd64.exe" + retry -retries 5 -scriptBlock { + Invoke-WebRequest -UseBasicParsing -Uri $pyInstallerUrl -OutFile $pyDownloadPath + } + Start-Process -FilePath $pyDownloadPath -ArgumentList "/quiet", "InstallAllUsers=1", "PrependPath=1", "Include_test=0" -Wait + $pyBinPath = "${env:ProgramFiles}\Python311" + $env:Path += ";$pyBinPath" python --version } + function withMinGW { - Write-Host "-- Install MinGW --" - choco install mingw -y - refreshenv + Write-Host "-- Installing MinGW --" + [Net.ServicePointManager]::SecurityProtocol = "tls11, tls12, ssl3" + $gwInstallerUrl = "https://github.com/brechtsanders/winlibs_mingw/releases/download/12.1.0-14.0.6-10.0.0-ucrt-r3/winlibs-x86_64-posix-seh-gcc-12.1.0-llvm-14.0.6-mingw-w64ucrt-10.0.0-r3.zip" + $gwInstallerCheckSumUrl = "$gwInstallerUrl.sha256" + $gwDownloadPath = "$env:TEMP\winlibs-x86_64.zip" + $gwDownloadCheckSumPath = "$env:TEMP\winlibs-x86_64.zip.sha256" + retry -retries 5 -scriptBlock { + Invoke-WebRequest -Uri $gwInstallerUrl -OutFile $gwDownloadPath + Invoke-WebRequest -Uri $gwInstallerCheckSumUrl -OutFile $gwDownloadCheckSumPath + } + $comparingResult = verifyFileChecksum -filePath $gwDownloadPath -checksumFilePath $gwDownloadCheckSumPath + if ($comparingResult) { + Expand-Archive -Path $gwDownloadPath -DestinationPath "$env:TEMP" + $gwBinPath = "$env:TEMP\mingw64\bin" + $env:Path += ";$gwBinPath" + } else { + exit 1 + } + } function installGoDependencies { $installPackages = @( @@ -46,26 +110,43 @@ function installGoDependencies { } } -fixCRLF +function withNmap($version) { + Write-Host "-- Installing Nmap $version --" + [Net.ServicePointManager]::SecurityProtocol = "tls, tls11, tls12, ssl3" + $nmapInstallerUrl = "https://nmap.org/dist/nmap-$version-setup.exe" + $nmapDownloadPath = "$env:TEMP\nmap-$version-setup.exe" + retry -retries 5 -scriptBlock { + Invoke-WebRequest -UseBasicParsing -Uri $nmapInstallerUrl -OutFile $nmapDownloadPath + } + Start-Process -FilePath $nmapDownloadPath -ArgumentList "/S" -Wait +} -withChoco +fixCRLF withGolang $env:GO_VERSION -installGoDependencies - withPython $env:SETUP_WIN_PYTHON_VERSION withMinGW +if ($env:BUILDKITE_PIPELINE_SLUG -eq "beats-packetbeat") { + withNmap $env:NMAP_WIN_VERSION +} + $ErrorActionPreference = "Continue" # set +e -Push-Location $WorkFolder +Set-Location -Path $WorkFolder + +$magefile = "$WORKSPACE\$WorkFolder\.magefile" +$env:MAGEFILE_CACHE = $magefile New-Item -ItemType Directory -Force -Path "build" -mage build unitTest -Pop-Location +if ($env:BUILDKITE_PIPELINE_SLUG -eq "beats-xpack-libbeat") { + mage -w reader/etw build goUnitTest +} else { + mage build unitTest +} $EXITCODE=$LASTEXITCODE $ErrorActionPreference = "Stop" diff --git a/.buildkite/x-pack/pipeline.xpack.libbeat.yml b/.buildkite/x-pack/pipeline.xpack.libbeat.yml index 34321b61161b..01695fa4fb65 100644 --- a/.buildkite/x-pack/pipeline.xpack.libbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.libbeat.yml @@ -1,5 +1,50 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json +name: "beats-xpack-libbeat" + +env: + IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" + IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64" + IMAGE_WIN_10: "family/general-windows-10" + IMAGE_WIN_11: "family/general-windows-11" + IMAGE_WIN_2016: "family/core-windows-2016" + IMAGE_WIN_2019: "family/core-windows-2019" + IMAGE_WIN_2022: "family/core-windows-2022" + GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" + GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" + GCP_WIN_MACHINE_TYPE: "n2-standard-8" + AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + BEATS_PROJECT_NAME: "x-pack/libbeat" steps: - - label: "Example test" - command: echo "Hello!" + + - input: "Input Parameters" + key: "input-run-all-stages" + fields: + - select: "Packetbeat - run_xpack_libbeat" + key: "run_xpack_libbeat" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "false" + - select: "Packetbeat - run_xpack_libbeat_arm_tests" + key: "run_xpack_libbeat_arm_tests" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "false" + if: "build.source == 'ui'" + + - wait: ~ + if: "build.source == 'ui'" + allow_dependency_failure: false + + - label: ":linux: Load dynamic packetbeat pipeline" + key: "packetbeat-pipeline" + command: ".buildkite/scripts/generate_xpack_libbeat_pipeline.sh" + notify: + - github_commit_status: + context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" From cdea924e320f2794a258d0c981646cd57bdecf0a Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Tue, 20 Feb 2024 21:23:20 +0000 Subject: [PATCH 04/26] Fix MSI breaking change release note to apply to all Beats (#38076) (#38077) * Add MSI installer change to 8.12 release notes. * Fix location of MSI breaking change notice. (cherry picked from commit 7a756931334fb622db8b100e18452601829e3411) Co-authored-by: Craig MacKenzie --- CHANGELOG.asciidoc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index a2513350661f..a25cb6baddbf 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -72,7 +72,12 @@ If you are not using the Elasticsearch output, set `queue.mem.flush.timeout: 1s` ==== Breaking changes +*Affecting all Beats* + +- Windows MSI installers now store configuration in C:\Program Files instead of C:\ProgramData. https://github.com/elastic/elastic-stack-installers/pull/209 + *Heartbeat* + - Decrease the ES default timeout to 10 for the load monitor state requests. - Windows MSI installers now store configuration in C:\Program Files instead of C:\ProgramData. https://github.com/elastic/elastic-stack-installers/pull/209 From 8214f9f5902998971ade2eb6ed8720c3ef384554 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Chema=20Mart=C3=ADnez?= Date: Wed, 21 Feb 2024 15:27:26 +0100 Subject: [PATCH 05/26] x-pack/filebeat/input/httpjson: Fix parseDate location offset (#37738) Add a new value template helper `parseDateInTZ` where users can apply the proper timezone when parsing dates, avoiding the limitations of `parseDate` with timezone abbreviations. It accepts numeric offsets and IANA time zone names. --- CHANGELOG.next.asciidoc | 1 + .../docs/inputs/input-httpjson.asciidoc | 3 +- x-pack/filebeat/input/httpjson/value_tpl.go | 53 +++++++++++++++++++ .../filebeat/input/httpjson/value_tpl_test.go | 35 ++++++++++++ 4 files changed, 91 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index a1c8df5256ca..14a24c00aabd 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -187,6 +187,7 @@ Setting environmental variable ELASTIC_NETINFO:false in Elastic Agent pod will d - Add ETW input. {pull}36915[36915] - Update CEL mito extensions to v1.9.0 to add keys/values helper. {pull}37971[37971] - Add logging for cache processor file reads and writes. {pull}38052[38052] +- Add parseDateInTZ value template for the HTTPJSON input {pull}37738[37738] *Auditbeat* diff --git a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc index cc3594780e4c..5fbd5dc15a5f 100644 --- a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc @@ -220,7 +220,8 @@ Some built-in helper functions are provided to work with the input state inside - `min`: returns the minimum of two values. - `mul`: multiplies two integers. - `now`: returns the current `time.Time` object in UTC. Optionally, it can receive a `time.Duration` as a parameter. Example: `[[now (parseDuration "-1h")]]` returns the time at 1 hour before now. -- `parseDate`: parses a date string and returns a `time.Time` in UTC. By default the expected layout is `RFC3339` but optionally can accept any of the Golang predefined layouts or a custom one. Example: `[[ parseDate "2020-11-05T12:25:32Z" ]]`, `[[ parseDate "2020-11-05T12:25:32.1234567Z" "RFC3339Nano" ]]`, `[[ (parseDate "Thu Nov 5 12:25:32 +0000 2020" "Mon Jan _2 15:04:05 -0700 2006").UTC ]]`. +- `parseDate`: parses a date string and returns a `time.Time` in UTC. By default the expected layout is `RFC3339` but optionally can accept any of the Golang predefined layouts or a custom one. Note: Parsing timezone abbreviations may cause ambiguities. Prefer `parseDateInTZ` for explicit timezone handling. Example: `[[ parseDate "2020-11-05T12:25:32Z" ]]`, `[[ parseDate "2020-11-05T12:25:32.1234567Z" "RFC3339Nano" ]]`, `[[ (parseDate "Thu Nov 5 12:25:32 +0000 2020" "Mon Jan _2 15:04:05 -0700 2006").UTC ]]`. +- `parseDateInTZ`: parses a date string within a specified timezone (TZ), returning a `time.Time` in UTC. Specified timezone overwrites implicit timezone from the input date. Accepts timezone offsets ("-07:00", "-0700", "-07") or IANA Time Zone names ("America/New_York"). If TZ is invalid, defaults to UTC. Optional layout argument as in parseDate. Example: `[[ parseDateInTZ "2020-11-05T12:25:32" "America/New_York" ]]`, `[[ parseDateInTZ "2020-11-05T12:25:32" "-07:00" "RFC3339" ]]`. - `parseDuration`: parses duration strings and returns `time.Duration`. Example: `[[parseDuration "1h"]]`. - `parseTimestampMilli`: parses a timestamp in milliseconds and returns a `time.Time` in UTC. Example: `[[parseTimestamp 1604582732000]]` returns `2020-11-05 13:25:32 +0000 UTC`. - `parseTimestampNano`: parses a timestamp in nanoseconds and returns a `time.Time` in UTC. Example: `[[parseTimestamp 1604582732000000000]]` returns `2020-11-05 13:25:32 +0000 UTC`. diff --git a/x-pack/filebeat/input/httpjson/value_tpl.go b/x-pack/filebeat/input/httpjson/value_tpl.go index 97bc75a62d94..cf7e43cf8e4c 100644 --- a/x-pack/filebeat/input/httpjson/value_tpl.go +++ b/x-pack/filebeat/input/httpjson/value_tpl.go @@ -71,6 +71,7 @@ func (t *valueTpl) Unpack(in string) error { "mul": mul, "now": now, "parseDate": parseDate, + "parseDateInTZ": parseDateInTZ, "parseDuration": parseDuration, "parseTimestamp": parseTimestamp, "parseTimestampMilli": parseTimestampMilli, @@ -194,6 +195,58 @@ func parseDate(date string, layout ...string) time.Time { return t.UTC() } +// parseDateInTZ parses a date string within a specified timezone, returning a time.Time +// 'tz' is the timezone (offset or IANA name) for parsing +func parseDateInTZ(date string, tz string, layout ...string) time.Time { + var ly string + if len(layout) == 0 { + ly = defaultTimeLayout + } else { + ly = layout[0] + } + if found := predefinedLayouts[ly]; found != "" { + ly = found + } + + var loc *time.Location + // Attempt to parse timezone as offset in various formats + for _, format := range []string{"-07", "-0700", "-07:00"} { + t, err := time.Parse(format, tz) + if err != nil { + continue + } + name, offset := t.Zone() + loc = time.FixedZone(name, offset) + break + } + + // If parsing tz as offset fails, try loading location by name + if loc == nil { + var err error + loc, err = time.LoadLocation(tz) + if err != nil { + loc = time.UTC // Default to UTC on error + } + } + + // Using Parse allows us not to worry about the timezone + // as the predefined timezone is applied afterwards + t, err := time.Parse(ly, date) + if err != nil { + return time.Time{} + } + + // Manually create a new time object with the parsed date components and the desired location + // It allows interpreting the parsed time in the specified timezone + year, month, day := t.Date() + hour, min, sec := t.Clock() + nanosec := t.Nanosecond() + localTime := time.Date(year, month, day, hour, min, sec, nanosec, loc) + + // Convert the time to UTC to standardize the output + return localTime.UTC() +} + func formatDate(date time.Time, layouttz ...string) string { var layout, tz string switch { diff --git a/x-pack/filebeat/input/httpjson/value_tpl_test.go b/x-pack/filebeat/input/httpjson/value_tpl_test.go index 487451099ad6..4b642a169735 100644 --- a/x-pack/filebeat/input/httpjson/value_tpl_test.go +++ b/x-pack/filebeat/input/httpjson/value_tpl_test.go @@ -142,6 +142,41 @@ func TestValueTpl(t *testing.T) { paramTr: transformable{}, expectedVal: "2020-11-05 12:25:32 +0000 UTC", }, + { + name: "func parseDateInTZ with RFC3339Nano and timezone offset", + value: `[[ parseDateInTZ "2020-11-05T12:25:32.1234567Z" "-0700" "RFC3339Nano" ]]`, + paramCtx: emptyTransformContext(), + paramTr: transformable{}, + expectedVal: "2020-11-05 19:25:32.1234567 +0000 UTC", + }, + { + name: "func parseDateInTZ defaults to RFC3339 with implicit offset and timezone", + value: `[[ parseDateInTZ "2020-11-05T12:25:32+04:00" "-0700" ]]`, + paramCtx: emptyTransformContext(), + paramTr: transformable{}, + expectedVal: "2020-11-05 19:25:32 +0000 UTC", + }, + { + name: "func parseDateInTZ defaults to RFC3339 with IANA timezone", + value: `[[ parseDateInTZ "2020-11-05T12:25:32Z" "America/New_York" ]]`, + paramCtx: emptyTransformContext(), + paramTr: transformable{}, + expectedVal: "2020-11-05 17:25:32 +0000 UTC", + }, + { + name: "func parseDateInTZ with custom layout and timezone name", + value: `[[ parseDateInTZ "Thu Nov 5 12:25:32 2020" "Europe/Paris" "Mon Jan _2 15:04:05 2006" ]]`, + paramCtx: emptyTransformContext(), + paramTr: transformable{}, + expectedVal: "2020-11-05 11:25:32 +0000 UTC", + }, + { + name: "func parseDateInTZ with invalid timezone", + value: `[[ parseDateInTZ "2020-11-05T12:25:32Z" "Invalid/Timezone" ]]`, + paramCtx: emptyTransformContext(), + paramTr: transformable{}, + expectedVal: "2020-11-05 12:25:32 +0000 UTC", + }, { name: "func formatDate", setup: func() { timeNow = func() time.Time { return time.Unix(1604582732, 0).UTC() } }, From 85e4e4693363262dbf699d71bbf814910432f6a3 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Wed, 21 Feb 2024 09:54:23 -0500 Subject: [PATCH 06/26] [Filebeat] ETW input - use errgroup (#38009) Use errgroup to wait on the ETW consumer routine. Use sync.OnceFunc to wrap the Close() func for the ETW session. Clarify a few log messages (follow-up to #36915) - https://github.com/elastic/beats/pull/36915#discussion_r1486467260 - https://github.com/elastic/beats/pull/36915#discussion_r1486467260 --- x-pack/filebeat/input/etw/input.go | 95 +++++++++++-------------- x-pack/filebeat/input/etw/input_test.go | 17 +++-- x-pack/libbeat/reader/etw/session.go | 4 +- 3 files changed, 51 insertions(+), 65 deletions(-) diff --git a/x-pack/filebeat/input/etw/input.go b/x-pack/filebeat/input/etw/input.go index 050fcf6ddf9b..b5b331b3c923 100644 --- a/x-pack/filebeat/input/etw/input.go +++ b/x-pack/filebeat/input/etw/input.go @@ -22,6 +22,7 @@ import ( "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" + "golang.org/x/sync/errgroup" "golang.org/x/sys/windows" ) @@ -66,6 +67,7 @@ type etwInput struct { log *logp.Logger config config etwSession *etw.Session + publisher stateless.Publisher operator sessionOperator } @@ -105,10 +107,13 @@ func (e *etwInput) Run(ctx input.Context, publisher stateless.Publisher) error { if err != nil { return fmt.Errorf("error initializing ETW session: %w", err) } + e.etwSession.Callback = e.consumeEvent + e.publisher = publisher // Set up logger with session information e.log = ctx.Logger.With("session", e.etwSession.Name) e.log.Info("Starting " + inputName + " input") + defer e.log.Info(inputName + " input stopped") // Handle realtime session creation or attachment if e.etwSession.Realtime { @@ -125,71 +130,31 @@ func (e *etwInput) Run(ctx input.Context, publisher stateless.Publisher) error { if err != nil { return fmt.Errorf("realtime session could not be created: %w", err) } - e.log.Debug("created session") + e.log.Debug("created new session") } } - // Defer the cleanup and closing of resources - var wg sync.WaitGroup - var once sync.Once - // Create an error channel to communicate errors from the goroutine - errChan := make(chan error, 1) + stopConsumer := sync.OnceFunc(e.Close) + defer stopConsumer() - defer func() { - once.Do(e.Close) - e.log.Info(inputName + " input stopped") + // Stop the consumer upon input cancellation (shutdown). + go func() { + <-ctx.Cancelation.Done() + stopConsumer() }() - // eventReceivedCallback processes each ETW event - eventReceivedCallback := func(record *etw.EventRecord) uintptr { - if record == nil { - e.log.Error("received null event record") - return 1 - } - - e.log.Debugf("received event %d with length %d", record.EventHeader.EventDescriptor.Id, record.UserDataLength) - - data, err := etw.GetEventProperties(record) - if err != nil { - e.log.Errorw("failed to read event properties", "error", err) - return 1 - } - - evt := buildEvent(data, record.EventHeader, e.etwSession, e.config) - publisher.Publish(evt) - - return 0 - } - - // Set the callback function for the ETW session - e.etwSession.Callback = eventReceivedCallback - // Start a goroutine to consume ETW events - wg.Add(1) - go func() { - defer wg.Done() - e.log.Debug("starting to listen ETW events") + g := new(errgroup.Group) + g.Go(func() error { + e.log.Debug("starting ETW consumer") + defer e.log.Debug("stopped ETW consumer") if err = e.operator.startConsumer(e.etwSession); err != nil { - errChan <- fmt.Errorf("failed to start consumer: %w", err) // Send error to channel - return + return fmt.Errorf("failed running ETW consumer: %w", err) } - e.log.Debug("stopped to read ETW events from session") - errChan <- nil - }() + return nil + }) - // We ensure resources are closed when receiving a cancellation signal - go func() { - <-ctx.Cancelation.Done() - once.Do(e.Close) - }() - - wg.Wait() // Ensure all goroutines have finished before closing - close(errChan) - if err, ok := <-errChan; ok && err != nil { - return err - } - - return nil + return g.Wait() } var ( @@ -271,6 +236,26 @@ func convertFileTimeToGoTime(fileTime64 uint64) time.Time { return time.Unix(0, fileTime.Nanoseconds()).UTC() } +func (e *etwInput) consumeEvent(record *etw.EventRecord) uintptr { + if record == nil { + e.log.Error("received null event record") + return 1 + } + + e.log.Debugf("received event with ID %d and user-data length %d", record.EventHeader.EventDescriptor.Id, record.UserDataLength) + + data, err := etw.GetEventProperties(record) + if err != nil { + e.log.Errorw("failed to read event properties", "error", err) + return 1 + } + + evt := buildEvent(data, record.EventHeader, e.etwSession, e.config) + e.publisher.Publish(evt) + + return 0 +} + // Close stops the ETW session and logs the outcome. func (e *etwInput) Close() { if err := e.operator.stopSession(e.etwSession); err != nil { diff --git a/x-pack/filebeat/input/etw/input_test.go b/x-pack/filebeat/input/etw/input_test.go index af1fa36d4bd5..fd2673278d37 100644 --- a/x-pack/filebeat/input/etw/input_test.go +++ b/x-pack/filebeat/input/etw/input_test.go @@ -107,7 +107,8 @@ func Test_RunEtwInput_AttachToExistingSessionError(t *testing.T) { mockSession := &etw.Session{ Name: "MySession", Realtime: true, - NewSession: false} + NewSession: false, + } return mockSession, nil } // Setup the mock behavior for AttachToExistingSession @@ -146,7 +147,8 @@ func Test_RunEtwInput_CreateRealtimeSessionError(t *testing.T) { mockSession := &etw.Session{ Name: "MySession", Realtime: true, - NewSession: true} + NewSession: true, + } return mockSession, nil } // Setup the mock behavior for AttachToExistingSession @@ -189,7 +191,8 @@ func Test_RunEtwInput_StartConsumerError(t *testing.T) { mockSession := &etw.Session{ Name: "MySession", Realtime: true, - NewSession: true} + NewSession: true, + } return mockSession, nil } // Setup the mock behavior for AttachToExistingSession @@ -232,7 +235,7 @@ func Test_RunEtwInput_StartConsumerError(t *testing.T) { // Run test err := etwInput.Run(inputCtx, nil) - assert.EqualError(t, err, "failed to start consumer: mock error") + assert.EqualError(t, err, "failed running ETW consumer: mock error") } func Test_RunEtwInput_Success(t *testing.T) { @@ -244,7 +247,8 @@ func Test_RunEtwInput_Success(t *testing.T) { mockSession := &etw.Session{ Name: "MySession", Realtime: true, - NewSession: true} + NewSession: true, + } return mockSession, nil } // Setup the mock behavior for AttachToExistingSession @@ -471,7 +475,6 @@ func Test_buildEvent(t *testing.T) { assert.Equal(t, tt.expected["event.severity"], mapEv["event.severity"]) assert.Equal(t, tt.expected["log.file.path"], mapEv["log.file.path"]) assert.Equal(t, tt.expected["log.level"], mapEv["log.level"]) - }) } } @@ -495,7 +498,7 @@ func Test_convertFileTimeToGoTime(t *testing.T) { { name: "TestActualDate", fileTime: 133515900000000000, // February 05, 2024, 7:00:00 AM - want: time.Date(2024, 02, 05, 7, 0, 0, 0, time.UTC), + want: time.Date(2024, 0o2, 0o5, 7, 0, 0, 0, time.UTC), }, } diff --git a/x-pack/libbeat/reader/etw/session.go b/x-pack/libbeat/reader/etw/session.go index 3216ff3af050..9d78d279de2d 100644 --- a/x-pack/libbeat/reader/etw/session.go +++ b/x-pack/libbeat/reader/etw/session.go @@ -229,10 +229,8 @@ func (s *Session) StartConsumer() error { // Open an ETW trace processing handle for consuming events // from an ETW real-time trace session or an ETW log file. s.traceHandler, err = s.openTrace(&elf) - switch { case err == nil: - // Handle specific errors for trace opening. case errors.Is(err, ERROR_BAD_PATHNAME): return fmt.Errorf("invalid log source when opening trace: %w", err) @@ -241,10 +239,10 @@ func (s *Session) StartConsumer() error { default: return fmt.Errorf("failed to open trace: %w", err) } + // Process the trace. This function blocks until processing ends. if err := s.processTrace(&s.traceHandler, 1, nil, nil); err != nil { return fmt.Errorf("failed to process trace: %w", err) } - return nil } From b1c0c15c384ed15438036ef317400e8dc41a3f39 Mon Sep 17 00:00:00 2001 From: Kuni Sen <30574753+kunisen@users.noreply.github.com> Date: Thu, 22 Feb 2024 15:54:51 +0900 Subject: [PATCH 07/26] Update protocol-metrics-packetbeat.asciidoc (#38096) Use `/inputs/` instead of `/inputs` --- packetbeat/docs/protocol-metrics-packetbeat.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packetbeat/docs/protocol-metrics-packetbeat.asciidoc b/packetbeat/docs/protocol-metrics-packetbeat.asciidoc index d36cce38e445..1b84eeb37b64 100644 --- a/packetbeat/docs/protocol-metrics-packetbeat.asciidoc +++ b/packetbeat/docs/protocol-metrics-packetbeat.asciidoc @@ -2,7 +2,7 @@ === Protocol-Specific Metrics Packetbeat exposes per-protocol metrics under the <>. -These metrics are exposed under the `/inputs` path. They can be used to +These metrics are exposed under the `/inputs/` path. They can be used to observe the activity of Packetbeat for the monitored protocol. [float] From 8ea54119e556fe9de613b7f15a14dbfdd2f31f84 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Thu, 22 Feb 2024 10:57:29 -0500 Subject: [PATCH 08/26] docs: Prepare Changelog for 8.12.2 (#38072) (#38100) * docs: Close changelog for 8.12.2 * Update CHANGELOG.asciidoc * Update CHANGELOG.asciidoc * Fix capitalization in CHANGELOG.asciidoc Co-authored-by: David Kilfoyle <41695641+kilfoyle@users.noreply.github.com> --------- Co-authored-by: Pierre HILBERT Co-authored-by: Craig MacKenzie Co-authored-by: David Kilfoyle <41695641+kilfoyle@users.noreply.github.com> (cherry picked from commit ae2cf677af9913c3e0f33877be7337a0258e70d7) Co-authored-by: Elastic Machine --- CHANGELOG.asciidoc | 22 ++++++++++++++++++++++ CHANGELOG.next.asciidoc | 5 +++-- libbeat/docs/release.asciidoc | 1 + 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index a25cb6baddbf..611b3664c070 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -3,6 +3,28 @@ :issue: https://github.com/elastic/beats/issues/ :pull: https://github.com/elastic/beats/pull/ +[[release-notes-8.12.2]] +=== Beats version 8.12.2 +https://github.com/elastic/beats/compare/v8.12.1\...v8.12.2[View commits] + +==== Bugfixes + +*Filebeat* + +- [threatintel] MISP pagination fixes. {pull}37898[37898] +- Fix file handle leak when handling errors in filestream. {pull}37973[37973] + +*Packetbeat* + +- Fix interface device parsing for packetbeat protocols. {pull}37946[37946] + +==== Added + +*Metricbeat* + +- Update `getOpTimestamp` in `replstatus` to fix sort and temp files generation issue in MongoDB. {pull}37688[37688] + + [[release-notes-8.12.1]] === Beats version 8.12.1 https://github.com/elastic/beats/compare/v8.12.0\...v8.12.1[View commits] diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 14a24c00aabd..f56ec5b48925 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -107,7 +107,6 @@ fields added to events containing the Beats version. {pull}37553[37553] *Packetbeat* -- Fix interface device parsing for packetbeat protocols. {pull}37946[37946] *Winlogbeat* @@ -218,7 +217,6 @@ Setting environmental variable ELASTIC_NETINFO:false in Elastic Agent pod will d *Metricbeat* -- Update `getOpTimestamp` in `replstatus` to fix sort and temp files generation issue in mongodb. {pull}37688[37688] *Osquerybeat* @@ -318,6 +316,9 @@ Setting environmental variable ELASTIC_NETINFO:false in Elastic Agent pod will d + + + diff --git a/libbeat/docs/release.asciidoc b/libbeat/docs/release.asciidoc index 08da0875d41e..55b9495a048c 100644 --- a/libbeat/docs/release.asciidoc +++ b/libbeat/docs/release.asciidoc @@ -8,6 +8,7 @@ This section summarizes the changes in each release. Also read <> for more detail about changes that affect upgrade. +* <> * <> * <> * <> From d7620710a4cd4698522421071245ffdd7c19a891 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Thu, 22 Feb 2024 18:02:57 +0100 Subject: [PATCH 09/26] Make condition work with numeric values as strings. (#38080) --- CHANGELOG.next.asciidoc | 1 + libbeat/conditions/range.go | 30 ++++++-------------------- libbeat/conditions/range_test.go | 14 ++++++------ libbeat/docs/processors-using.asciidoc | 2 +- 4 files changed, 16 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index f56ec5b48925..79e2b92f65a3 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -132,6 +132,7 @@ Setting environmental variable ELASTIC_NETINFO:false in Elastic Agent pod will d - Upgrade to elastic-agent-libs v0.7.3 and golang.org/x/crypto v0.17.0. {pull}37544[37544] - Make more selective the Pod autodiscovery upon node and namespace update events. {issue}37338[37338] {pull}37431[37431] - Upgrade go-sysinfo from 1.12.0 to 1.13.1. {pull}37996[37996] +- Make `range` condition work with numeric values as strings. {pull}38080[38080] *Auditbeat* diff --git a/libbeat/conditions/range.go b/libbeat/conditions/range.go index 39eb05031e12..cb5e75603fda 100644 --- a/libbeat/conditions/range.go +++ b/libbeat/conditions/range.go @@ -19,7 +19,6 @@ package conditions import ( "fmt" - "reflect" "strings" "github.com/elastic/elastic-agent-libs/logp" @@ -113,30 +112,13 @@ func (c Range) Check(event ValuesMap) bool { return false } - switch value.(type) { - case int, int8, int16, int32, int64: - intValue := reflect.ValueOf(value).Int() - - if !checkValue(float64(intValue), rangeValue) { - return false - } - - case uint, uint8, uint16, uint32, uint64: - uintValue := reflect.ValueOf(value).Uint() - - if !checkValue(float64(uintValue), rangeValue) { - return false - } - - case float64, float32: - floatValue := reflect.ValueOf(value).Float() - - if !checkValue(floatValue, rangeValue) { - return false - } + floatValue, err := ExtractFloat(value) + if err != nil { + logp.L().Named(logName).Warnf(err.Error()) + return false + } - default: - logp.L().Named(logName).Warnf("unexpected type %T in range condition.", value) + if !checkValue(floatValue, rangeValue) { return false } diff --git a/libbeat/conditions/range_test.go b/libbeat/conditions/range_test.go index cbf4db379095..808c3456a47c 100644 --- a/libbeat/conditions/range_test.go +++ b/libbeat/conditions/range_test.go @@ -83,7 +83,8 @@ func TestMultipleOpenRangeConditionNegativeMatch(t *testing.T) { var procCPURangeConfig = &Config{ Range: &Fields{fields: map[string]interface{}{ - "proc.cpu.total_p.gte": 0.5, + "proc.cpu.total_p.gte": 0.5, + "proc.cpu.total_p_str.gte": 0.5, }}, } @@ -94,11 +95,12 @@ func TestOpenGteRangeConditionPositiveMatch(t *testing.T) { "proc": mapstr.M{ "cmdline": "/System/Library/Frameworks/CoreServices.framework/Frameworks/Metadata.framework/Versions/A/Support/mdworker -s mdworker -c MDSImporterWorker -m com.apple.mdworker.single", "cpu": mapstr.M{ - "start_time": "09:19", - "system": 22, - "total": 66, - "total_p": 0.6, - "user": 44, + "start_time": "09:19", + "system": 22, + "total": 66, + "total_p_str": "0.6", + "total_p": 0.6, + "user": 44, }, "name": "mdworker", "pid": 44978, diff --git a/libbeat/docs/processors-using.asciidoc b/libbeat/docs/processors-using.asciidoc index eb5f391a558e..dd91ea8d5db6 100644 --- a/libbeat/docs/processors-using.asciidoc +++ b/libbeat/docs/processors-using.asciidoc @@ -276,7 +276,7 @@ regexp: The `range` condition checks if the field is in a certain range of values. The condition supports `lt`, `lte`, `gt` and `gte`. The condition accepts only -integer or float values. +integer, float, or strings that can be converted to either of these as values. For example, the following condition checks for failed HTTP transactions by comparing the `http.response.code` field with 400. From a0a567c36893bc39eb84cc80fee381ad82c4f072 Mon Sep 17 00:00:00 2001 From: Panos Koutsovasilis Date: Thu, 22 Feb 2024 20:13:28 +0200 Subject: [PATCH 10/26] [filebeat] add netflow input metrics (#38055) * feat: introduce input/netmetrics to allow multiple inputs utilise the same tcp and/or udp metrics * feat: add netflow input specific metrics discarded_events_total, flows_total, decode_errors_total * feat: add netflow input specific metric open_connections * fix: omit the name of unused receivers * fix: remove deprecated ioutil with io * fix: remove redundant type conversion * fix: address naked return * fix: proper equality check in TestSessionState * fix: ignore explicitly the error from registering v9 protocol * doc: update changelog * feat: refactor netmetrics to instantiate and expose monitoring registry * feat: rename GetRegistry to Registry in TCP and UDP of netmetrics to promote idiomatic go * doc: update input-netflow.asciidoc to capture the new metrics * fix: move metrics last in input-netflow.asciidoc --- CHANGELOG.next.asciidoc | 1 + .../procnet.go => netmetrics/netmetrics.go} | 59 +++- .../netmetrics_test.go} | 6 +- filebeat/input/netmetrics/tcp.go | 251 ++++++++++++++++ .../input_test.go => netmetrics/tcp_test.go} | 24 +- .../testdata/proc_net_tcp.txt | 0 .../testdata/proc_net_tcp6.txt | 0 .../testdata/proc_net_udp.txt | 0 .../testdata/proc_net_udp6.txt | 0 filebeat/input/netmetrics/udp.go | 267 +++++++++++++++++ .../input_test.go => netmetrics/udp_test.go} | 24 +- filebeat/input/tcp/input.go | 254 +--------------- filebeat/input/udp/input.go | 270 +----------------- .../docs/inputs/input-netflow.asciidoc | 28 ++ .../input/netflow/decoder/config/config.go | 36 ++- .../input/netflow/decoder/ipfix/decoder.go | 2 +- .../input/netflow/decoder/v9/decoder.go | 6 +- .../input/netflow/decoder/v9/session.go | 25 +- .../input/netflow/decoder/v9/session_test.go | 16 +- .../filebeat/input/netflow/decoder/v9/v9.go | 4 +- x-pack/filebeat/input/netflow/input.go | 138 ++++----- x-pack/filebeat/input/netflow/metrics.go | 55 ++++ 22 files changed, 801 insertions(+), 665 deletions(-) rename filebeat/input/{internal/procnet/procnet.go => netmetrics/netmetrics.go} (63%) rename filebeat/input/{internal/procnet/procnet_test.go => netmetrics/netmetrics_test.go} (92%) create mode 100644 filebeat/input/netmetrics/tcp.go rename filebeat/input/{tcp/input_test.go => netmetrics/tcp_test.go} (82%) rename filebeat/input/{tcp => netmetrics}/testdata/proc_net_tcp.txt (100%) rename filebeat/input/{tcp => netmetrics}/testdata/proc_net_tcp6.txt (100%) rename filebeat/input/{udp => netmetrics}/testdata/proc_net_udp.txt (100%) rename filebeat/input/{udp => netmetrics}/testdata/proc_net_udp6.txt (100%) create mode 100644 filebeat/input/netmetrics/udp.go rename filebeat/input/{udp/input_test.go => netmetrics/udp_test.go} (83%) create mode 100644 x-pack/filebeat/input/netflow/metrics.go diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 79e2b92f65a3..d7761c359dca 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -27,6 +27,7 @@ fields added to events containing the Beats version. {pull}37553[37553] *Filebeat* - Convert netflow input to API v2 and disable event normalisation {pull}37901[37901] +- Introduce input/netmetrics and refactor netflow input metrics {pull}38055[38055] *Heartbeat* diff --git a/filebeat/input/internal/procnet/procnet.go b/filebeat/input/netmetrics/netmetrics.go similarity index 63% rename from filebeat/input/internal/procnet/procnet.go rename to filebeat/input/netmetrics/netmetrics.go index 0761c9994c79..c320eb3fb476 100644 --- a/filebeat/input/internal/procnet/procnet.go +++ b/filebeat/input/netmetrics/netmetrics.go @@ -15,24 +15,27 @@ // specific language governing permissions and limitations // under the License. -// Package procnet provides support for obtaining and formatting /proc/net -// network addresses for linux systems. -package procnet +// Package netmetrics provides different metricsets for capturing network-related metrics, +// such as UDP and TCP metrics through NewUDP and NewTCP, respectively. +package netmetrics import ( + "bytes" + "encoding/hex" "fmt" "net" "strconv" + "strings" "github.com/elastic/elastic-agent-libs/logp" ) -// Addrs returns the linux /proc/net/tcp or /proc/net/udp addresses for the +// addrs returns the linux /proc/net/tcp or /proc/net/udp addresses for the // provided host address, addr. addr is a host:port address and host may be // an IPv4 or IPv6 address, or an FQDN for the host. The returned slices // contain the string representations of the address as they would appear in // the /proc/net tables. -func Addrs(addr string, log *logp.Logger) (addr4, addr6 []string, err error) { +func addrs(addr string, log *logp.Logger) (addr4, addr6 []string, err error) { host, port, err := net.SplitHostPort(addr) if err != nil { return nil, nil, fmt.Errorf("failed to get address for %s: could not split host and port: %w", addr, err) @@ -54,9 +57,9 @@ func Addrs(addr string, log *logp.Logger) (addr4, addr6 []string, err error) { // the len constants all addresses may appear to be IPv6. switch { case len(p.To4()) == net.IPv4len: - addr4 = append(addr4, IPv4(p, int(pn))) + addr4 = append(addr4, ipV4(p, int(pn))) case len(p.To16()) == net.IPv6len: - addr6 = append(addr6, IPv6(p, int(pn))) + addr6 = append(addr6, ipV6(p, int(pn))) default: log.Warnf("unexpected addr length %d for %s", len(p), p) } @@ -64,13 +67,13 @@ func Addrs(addr string, log *logp.Logger) (addr4, addr6 []string, err error) { return addr4, addr6, nil } -// IPv4 returns the string representation of an IPv4 address in a /proc/net table. -func IPv4(ip net.IP, port int) string { +// ipV4 returns the string representation of an IPv4 address in a /proc/net table. +func ipV4(ip net.IP, port int) string { return fmt.Sprintf("%08X:%04X", reverse(ip.To4()), port) } -// IPv6 returns the string representation of an IPv6 address in a /proc/net table. -func IPv6(ip net.IP, port int) string { +// ipV6 returns the string representation of an IPv6 address in a /proc/net table. +func ipV6(ip net.IP, port int) string { return fmt.Sprintf("%032X:%04X", reverse(ip.To16()), port) } @@ -81,3 +84,37 @@ func reverse(b []byte) []byte { } return c } + +func contains(b []byte, addr []string, addrIsUnspecified []bool) bool { + for i, a := range addr { + if addrIsUnspecified[i] { + _, ap, pok := strings.Cut(a, ":") + _, bp, bok := bytes.Cut(b, []byte(":")) + if pok && bok && strings.EqualFold(string(bp), ap) { + return true + } + } else if strings.EqualFold(string(b), a) { + return true + } + } + return false +} + +func containsUnspecifiedAddr(addr []string) (yes bool, which []bool, bad []string) { + which = make([]bool, len(addr)) + for i, a := range addr { + prefix, _, ok := strings.Cut(a, ":") + if !ok { + continue + } + ip, err := hex.DecodeString(prefix) + if err != nil { + bad = append(bad, a) + } + if net.IP(ip).IsUnspecified() { + yes = true + which[i] = true + } + } + return yes, which, bad +} diff --git a/filebeat/input/internal/procnet/procnet_test.go b/filebeat/input/netmetrics/netmetrics_test.go similarity index 92% rename from filebeat/input/internal/procnet/procnet_test.go rename to filebeat/input/netmetrics/netmetrics_test.go index 975f4a4aa958..c21b52c63ec0 100644 --- a/filebeat/input/internal/procnet/procnet_test.go +++ b/filebeat/input/netmetrics/netmetrics_test.go @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package procnet +package netmetrics import ( "testing" @@ -25,7 +25,7 @@ import ( func TestAddrs(t *testing.T) { t.Run("ipv4", func(t *testing.T) { - addr4, addr6, err := Addrs("0.0.0.0:9001", logp.L()) + addr4, addr6, err := addrs("0.0.0.0:9001", logp.L()) if err != nil { t.Errorf("unexpected error: %v", err) } @@ -38,7 +38,7 @@ func TestAddrs(t *testing.T) { }) t.Run("ipv6", func(t *testing.T) { - addr4, addr6, err := Addrs("[::]:9001", logp.L()) + addr4, addr6, err := addrs("[::]:9001", logp.L()) if err != nil { t.Errorf("unexpected error: %v", err) } diff --git a/filebeat/input/netmetrics/tcp.go b/filebeat/input/netmetrics/tcp.go new file mode 100644 index 000000000000..19accfc2500e --- /dev/null +++ b/filebeat/input/netmetrics/tcp.go @@ -0,0 +1,251 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package netmetrics + +import ( + "bytes" + "errors" + "fmt" + "os" + "runtime" + "strconv" + "time" + + "github.com/rcrowley/go-metrics" + + "github.com/elastic/beats/v7/libbeat/monitoring/inputmon" + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/monitoring" + "github.com/elastic/elastic-agent-libs/monitoring/adapter" +) + +// TCP handles the TCP metric reporting. +type TCP struct { + unregister func() + done chan struct{} + + monitorRegistry *monitoring.Registry + + lastPacket time.Time + + device *monitoring.String // name of the device being monitored + packets *monitoring.Uint // number of packets processed + bytes *monitoring.Uint // number of bytes processed + rxQueue *monitoring.Uint // value of the rx_queue field from /proc/net/tcp{,6} (only on linux systems) + arrivalPeriod metrics.Sample // histogram of the elapsed time between packet arrivals + processingTime metrics.Sample // histogram of the elapsed time between packet receipt and publication +} + +// NewTCP returns a new TCP input metricset. Note that if the id is empty then a nil TCP metricset is returned. +func NewTCP(inputName string, id string, device string, poll time.Duration, log *logp.Logger) *TCP { + if id == "" { + return nil + } + reg, unreg := inputmon.NewInputRegistry(inputName, id, nil) + out := &TCP{ + unregister: unreg, + monitorRegistry: reg, + device: monitoring.NewString(reg, "device"), + packets: monitoring.NewUint(reg, "received_events_total"), + bytes: monitoring.NewUint(reg, "received_bytes_total"), + rxQueue: monitoring.NewUint(reg, "receive_queue_length"), + arrivalPeriod: metrics.NewUniformSample(1024), + processingTime: metrics.NewUniformSample(1024), + } + _ = adapter.NewGoMetrics(reg, "arrival_period", adapter.Accept). + Register("histogram", metrics.NewHistogram(out.arrivalPeriod)) + _ = adapter.NewGoMetrics(reg, "processing_time", adapter.Accept). + Register("histogram", metrics.NewHistogram(out.processingTime)) + + out.device.Set(device) + + if poll > 0 && runtime.GOOS == "linux" { + addr4, addr6, err := addrs(device, log) + if err != nil { + log.Warn(err) + return out + } + out.done = make(chan struct{}) + go out.poll(addr4, addr6, poll, log) + } + + return out +} + +// Log logs metric for the given packet. +func (m *TCP) Log(data []byte, timestamp time.Time) { + if m == nil { + return + } + m.processingTime.Update(time.Since(timestamp).Nanoseconds()) + m.packets.Add(1) + m.bytes.Add(uint64(len(data))) + if !m.lastPacket.IsZero() { + m.arrivalPeriod.Update(timestamp.Sub(m.lastPacket).Nanoseconds()) + } + m.lastPacket = timestamp +} + +// poll periodically gets TCP buffer stats from the OS. +func (m *TCP) poll(addr, addr6 []string, each time.Duration, log *logp.Logger) { + hasUnspecified, addrIsUnspecified, badAddr := containsUnspecifiedAddr(addr) + if badAddr != nil { + log.Warnf("failed to parse IPv4 addrs for metric collection %q", badAddr) + } + hasUnspecified6, addrIsUnspecified6, badAddr := containsUnspecifiedAddr(addr6) + if badAddr != nil { + log.Warnf("failed to parse IPv6 addrs for metric collection %q", badAddr) + } + + // Do an initial check for access to the filesystem and of the + // value constructed by containsUnspecifiedAddr. This gives a + // base level for the rx_queue values and ensures that if the + // constructed address values are malformed we panic early + // within the period of system testing. + want4 := true + rx, err := procNetTCP("/proc/net/tcp", addr, hasUnspecified, addrIsUnspecified) + if err != nil { + want4 = false + log.Infof("did not get initial tcp stats from /proc: %v", err) + } + want6 := true + rx6, err := procNetTCP("/proc/net/tcp6", addr6, hasUnspecified6, addrIsUnspecified6) + if err != nil { + want6 = false + log.Infof("did not get initial tcp6 stats from /proc: %v", err) + } + if !want4 && !want6 { + log.Warnf("failed to get initial tcp or tcp6 stats from /proc: %v", err) + } else { + m.rxQueue.Set(uint64(rx + rx6)) + } + + t := time.NewTicker(each) + for { + select { + case <-t.C: + var found bool + rx, err := procNetTCP("/proc/net/tcp", addr, hasUnspecified, addrIsUnspecified) + if err != nil { + if want4 { + log.Warnf("failed to get tcp stats from /proc: %v", err) + } + } else { + found = true + want4 = true + } + rx6, err := procNetTCP("/proc/net/tcp6", addr6, hasUnspecified6, addrIsUnspecified6) + if err != nil { + if want6 { + log.Warnf("failed to get tcp6 stats from /proc: %v", err) + } + } else { + found = true + want6 = true + } + if found { + m.rxQueue.Set(uint64(rx + rx6)) + } + case <-m.done: + t.Stop() + return + } + } +} + +// Registry returns the monitoring registry of the TCP metricset. +func (m *TCP) Registry() *monitoring.Registry { + if m == nil { + return nil + } + + return m.monitorRegistry +} + +// procNetTCP returns the rx_queue field of the TCP socket table for the +// socket on the provided address formatted in hex, xxxxxxxx:xxxx or the IPv6 +// equivalent. +// This function is only useful on linux due to its dependence on the /proc +// filesystem, but is kept in this file for simplicity. If hasUnspecified +// is true, all addresses listed in the file in path are considered, and the +// sum of rx_queue matching the addr ports is returned where the corresponding +// addrIsUnspecified is true. +func procNetTCP(path string, addr []string, hasUnspecified bool, addrIsUnspecified []bool) (rx int64, err error) { + if len(addr) == 0 { + return 0, nil + } + if len(addr) != len(addrIsUnspecified) { + return 0, errors.New("mismatched address/unspecified lists: please report this") + } + b, err := os.ReadFile(path) + if err != nil { + return 0, err + } + lines := bytes.Split(b, []byte("\n")) + if len(lines) < 2 { + return 0, fmt.Errorf("%s entry not found for %s (no line)", path, addr) + } + var found bool + for _, l := range lines[1:] { + f := bytes.Fields(l) + const queuesField = 4 + if len(f) > queuesField && contains(f[1], addr, addrIsUnspecified) { + _, r, ok := bytes.Cut(f[4], []byte(":")) + if !ok { + return 0, errors.New("no rx_queue field " + string(f[queuesField])) + } + found = true + + // queue lengths are hex, e.g.: + // - https://elixir.bootlin.com/linux/v6.2.11/source/net/ipv4/tcp_ipv4.c#L2643 + // - https://elixir.bootlin.com/linux/v6.2.11/source/net/ipv6/tcp_ipv6.c#L1987 + v, err := strconv.ParseInt(string(r), 16, 64) + if err != nil { + return 0, fmt.Errorf("failed to parse rx_queue: %w", err) + } + rx += v + + if hasUnspecified { + continue + } + return rx, nil + } + } + if found { + return rx, nil + } + return 0, fmt.Errorf("%s entry not found for %s", path, addr) +} + +// Close closes the TCP metricset and unregister the metrics. +func (m *TCP) Close() { + if m == nil { + return + } + if m.done != nil { + // Shut down poller and wait until done before unregistering metrics. + m.done <- struct{}{} + } + + if m.unregister != nil { + m.unregister() + m.unregister = nil + } + + m.monitorRegistry = nil +} diff --git a/filebeat/input/tcp/input_test.go b/filebeat/input/netmetrics/tcp_test.go similarity index 82% rename from filebeat/input/tcp/input_test.go rename to filebeat/input/netmetrics/tcp_test.go index 5f53de516827..9e9001dda8ad 100644 --- a/filebeat/input/tcp/input_test.go +++ b/filebeat/input/netmetrics/tcp_test.go @@ -15,22 +15,20 @@ // specific language governing permissions and limitations // under the License. -package tcp +package netmetrics import ( "net" "testing" "github.com/stretchr/testify/assert" - - "github.com/elastic/beats/v7/filebeat/input/internal/procnet" ) func TestProcNetTCP(t *testing.T) { t.Run("IPv4", func(t *testing.T) { path := "testdata/proc_net_tcp.txt" t.Run("with_match", func(t *testing.T) { - addr := []string{procnet.IPv4(net.IP{0x7f, 0x00, 0x00, 0x01}, 0x17ac)} + addr := []string{ipV4(net.IP{0x7f, 0x00, 0x00, 0x01}, 0x17ac)} hasUnspecified, addrIsUnspecified, bad := containsUnspecifiedAddr(addr) rx, err := procNetTCP(path, addr, hasUnspecified, addrIsUnspecified) if err != nil { @@ -41,7 +39,7 @@ func TestProcNetTCP(t *testing.T) { }) t.Run("leading_zero", func(t *testing.T) { - addr := []string{procnet.IPv4(net.IP{0x00, 0x7f, 0x01, 0x00}, 0x17af)} + addr := []string{ipV4(net.IP{0x00, 0x7f, 0x01, 0x00}, 0x17af)} hasUnspecified, addrIsUnspecified, bad := containsUnspecifiedAddr(addr) rx, err := procNetTCP(path, addr, hasUnspecified, addrIsUnspecified) if err != nil { @@ -52,7 +50,7 @@ func TestProcNetTCP(t *testing.T) { }) t.Run("unspecified", func(t *testing.T) { - addr := []string{procnet.IPv4(net.ParseIP("0.0.0.0"), 0x17ac)} + addr := []string{ipV4(net.ParseIP("0.0.0.0"), 0x17ac)} hasUnspecified, addrIsUnspecified, bad := containsUnspecifiedAddr(addr) rx, err := procNetTCP(path, addr, hasUnspecified, addrIsUnspecified) if err != nil { @@ -64,8 +62,8 @@ func TestProcNetTCP(t *testing.T) { t.Run("without_match", func(t *testing.T) { addr := []string{ - procnet.IPv4(net.IP{0xde, 0xad, 0xbe, 0xef}, 0xf00d), - procnet.IPv4(net.IP{0xba, 0x1d, 0xfa, 0xce}, 0x1135), + ipV4(net.IP{0xde, 0xad, 0xbe, 0xef}, 0xf00d), + ipV4(net.IP{0xba, 0x1d, 0xfa, 0xce}, 0x1135), } hasUnspecified, addrIsUnspecified, bad := containsUnspecifiedAddr(addr) _, err := procNetTCP(path, addr, hasUnspecified, addrIsUnspecified) @@ -89,7 +87,7 @@ func TestProcNetTCP(t *testing.T) { t.Run("IPv6", func(t *testing.T) { path := "testdata/proc_net_tcp6.txt" t.Run("with_match", func(t *testing.T) { - addr := []string{procnet.IPv6(net.IP{0: 0x7f, 3: 0x01, 15: 0}, 0x17ac)} + addr := []string{ipV6(net.IP{0: 0x7f, 3: 0x01, 15: 0}, 0x17ac)} hasUnspecified, addrIsUnspecified, bad := containsUnspecifiedAddr(addr) rx, err := procNetTCP(path, addr, hasUnspecified, addrIsUnspecified) if err != nil { @@ -100,7 +98,7 @@ func TestProcNetTCP(t *testing.T) { }) t.Run("leading_zero", func(t *testing.T) { - addr := []string{procnet.IPv6(net.IP{1: 0x7f, 2: 0x01, 15: 0}, 0x17af)} + addr := []string{ipV6(net.IP{1: 0x7f, 2: 0x01, 15: 0}, 0x17af)} hasUnspecified, addrIsUnspecified, bad := containsUnspecifiedAddr(addr) rx, err := procNetTCP(path, addr, hasUnspecified, addrIsUnspecified) if err != nil { @@ -111,7 +109,7 @@ func TestProcNetTCP(t *testing.T) { }) t.Run("unspecified", func(t *testing.T) { - addr := []string{procnet.IPv6(net.ParseIP("[::]"), 0x17ac)} + addr := []string{ipV6(net.ParseIP("[::]"), 0x17ac)} hasUnspecified, addrIsUnspecified, bad := containsUnspecifiedAddr(addr) rx, err := procNetTCP(path, addr, hasUnspecified, addrIsUnspecified) if err != nil { @@ -123,8 +121,8 @@ func TestProcNetTCP(t *testing.T) { t.Run("without_match", func(t *testing.T) { addr := []string{ - procnet.IPv6(net.IP{0xde, 0xad, 0xbe, 0xef, 0xde, 0xad, 0xbe, 0xef, 0xde, 0xad, 0xbe, 0xef, 0xde, 0xad, 0xbe, 0xef}, 0xf00d), - procnet.IPv6(net.IP{0xba, 0x1d, 0xfa, 0xce, 0xba, 0x1d, 0xfa, 0xce, 0xba, 0x1d, 0xfa, 0xce, 0xba, 0x1d, 0xfa, 0xce}, 0x1135), + ipV6(net.IP{0xde, 0xad, 0xbe, 0xef, 0xde, 0xad, 0xbe, 0xef, 0xde, 0xad, 0xbe, 0xef, 0xde, 0xad, 0xbe, 0xef}, 0xf00d), + ipV6(net.IP{0xba, 0x1d, 0xfa, 0xce, 0xba, 0x1d, 0xfa, 0xce, 0xba, 0x1d, 0xfa, 0xce, 0xba, 0x1d, 0xfa, 0xce}, 0x1135), } hasUnspecified, addrIsUnspecified, bad := containsUnspecifiedAddr(addr) _, err := procNetTCP(path, addr, hasUnspecified, addrIsUnspecified) diff --git a/filebeat/input/tcp/testdata/proc_net_tcp.txt b/filebeat/input/netmetrics/testdata/proc_net_tcp.txt similarity index 100% rename from filebeat/input/tcp/testdata/proc_net_tcp.txt rename to filebeat/input/netmetrics/testdata/proc_net_tcp.txt diff --git a/filebeat/input/tcp/testdata/proc_net_tcp6.txt b/filebeat/input/netmetrics/testdata/proc_net_tcp6.txt similarity index 100% rename from filebeat/input/tcp/testdata/proc_net_tcp6.txt rename to filebeat/input/netmetrics/testdata/proc_net_tcp6.txt diff --git a/filebeat/input/udp/testdata/proc_net_udp.txt b/filebeat/input/netmetrics/testdata/proc_net_udp.txt similarity index 100% rename from filebeat/input/udp/testdata/proc_net_udp.txt rename to filebeat/input/netmetrics/testdata/proc_net_udp.txt diff --git a/filebeat/input/udp/testdata/proc_net_udp6.txt b/filebeat/input/netmetrics/testdata/proc_net_udp6.txt similarity index 100% rename from filebeat/input/udp/testdata/proc_net_udp6.txt rename to filebeat/input/netmetrics/testdata/proc_net_udp6.txt diff --git a/filebeat/input/netmetrics/udp.go b/filebeat/input/netmetrics/udp.go new file mode 100644 index 000000000000..de2344f1efd1 --- /dev/null +++ b/filebeat/input/netmetrics/udp.go @@ -0,0 +1,267 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package netmetrics + +import ( + "bytes" + "errors" + "fmt" + "os" + "runtime" + "strconv" + "time" + + "github.com/rcrowley/go-metrics" + + "github.com/elastic/beats/v7/libbeat/monitoring/inputmon" + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/monitoring" + "github.com/elastic/elastic-agent-libs/monitoring/adapter" +) + +// UDP captures UDP related metrics. +type UDP struct { + unregister func() + done chan struct{} + + monitorRegistry *monitoring.Registry + + lastPacket time.Time + + device *monitoring.String // name of the device being monitored + packets *monitoring.Uint // number of packets processed + bytes *monitoring.Uint // number of bytes processed + bufferLen *monitoring.Uint // configured read buffer length + rxQueue *monitoring.Uint // value of the rx_queue field from /proc/net/udp{,6} (only on linux systems) + drops *monitoring.Uint // number of udp drops noted in /proc/net/udp{,6} + arrivalPeriod metrics.Sample // histogram of the elapsed time between packet arrivals + processingTime metrics.Sample // histogram of the elapsed time between packet receipt and publication +} + +// NewUDP returns a new UDP input metricset. Note that if the id is empty then a nil UDP metricset is returned. +func NewUDP(inputName string, id string, device string, buflen uint64, poll time.Duration, log *logp.Logger) *UDP { + if id == "" { + return nil + } + reg, unreg := inputmon.NewInputRegistry(inputName, id, nil) + out := &UDP{ + unregister: unreg, + monitorRegistry: reg, + bufferLen: monitoring.NewUint(reg, "udp_read_buffer_length_gauge"), + device: monitoring.NewString(reg, "device"), + packets: monitoring.NewUint(reg, "received_events_total"), + bytes: monitoring.NewUint(reg, "received_bytes_total"), + rxQueue: monitoring.NewUint(reg, "receive_queue_length"), + drops: monitoring.NewUint(reg, "system_packet_drops"), + arrivalPeriod: metrics.NewUniformSample(1024), + processingTime: metrics.NewUniformSample(1024), + } + _ = adapter.NewGoMetrics(reg, "arrival_period", adapter.Accept). + Register("histogram", metrics.NewHistogram(out.arrivalPeriod)) + _ = adapter.NewGoMetrics(reg, "processing_time", adapter.Accept). + Register("histogram", metrics.NewHistogram(out.processingTime)) + + out.device.Set(device) + out.bufferLen.Set(buflen) + + if poll > 0 && runtime.GOOS == "linux" { + addr, addr6, err := addrs(device, log) + if err != nil { + log.Warn(err) + return out + } + out.done = make(chan struct{}) + go out.poll(addr, addr6, poll, log) + } + + return out +} + +// Log logs metric for the given packet. +func (m *UDP) Log(data []byte, timestamp time.Time) { + if m == nil { + return + } + m.processingTime.Update(time.Since(timestamp).Nanoseconds()) + m.packets.Add(1) + m.bytes.Add(uint64(len(data))) + if !m.lastPacket.IsZero() { + m.arrivalPeriod.Update(timestamp.Sub(m.lastPacket).Nanoseconds()) + } + m.lastPacket = timestamp +} + +// poll periodically gets UDP buffer and packet drops stats from the OS. +func (m *UDP) poll(addr, addr6 []string, each time.Duration, log *logp.Logger) { + hasUnspecified, addrIsUnspecified, badAddr := containsUnspecifiedAddr(addr) + if badAddr != nil { + log.Warnf("failed to parse IPv4 addrs for metric collection %q", badAddr) + } + hasUnspecified6, addrIsUnspecified6, badAddr := containsUnspecifiedAddr(addr6) + if badAddr != nil { + log.Warnf("failed to parse IPv6 addrs for metric collection %q", badAddr) + } + + // Do an initial check for access to the filesystem and of the + // value constructed by containsUnspecifiedAddr. This gives a + // base level for the rx_queue and drops values and ensures that + // if the constructed address values are malformed we panic early + // within the period of system testing. + want4 := true + rx, drops, err := procNetUDP("/proc/net/udp", addr, hasUnspecified, addrIsUnspecified) + if err != nil { + want4 = false + log.Infof("did not get initial udp stats from /proc: %v", err) + } + want6 := true + rx6, drops6, err := procNetUDP("/proc/net/udp6", addr6, hasUnspecified6, addrIsUnspecified6) + if err != nil { + want6 = false + log.Infof("did not get initial udp6 stats from /proc: %v", err) + } + if !want4 && !want6 { + log.Warnf("failed to get initial udp or udp6 stats from /proc: %v", err) + } else { + m.rxQueue.Set(uint64(rx + rx6)) + m.drops.Set(uint64(drops + drops6)) + } + + t := time.NewTicker(each) + for { + select { + case <-t.C: + var found bool + rx, drops, err := procNetUDP("/proc/net/udp", addr, hasUnspecified, addrIsUnspecified) + if err != nil { + if want4 { + log.Warnf("failed to get udp stats from /proc: %v", err) + } + } else { + found = true + want4 = true + } + rx6, drops6, err := procNetUDP("/proc/net/udp6", addr6, hasUnspecified6, addrIsUnspecified6) + if err != nil { + if want6 { + log.Warnf("failed to get udp6 stats from /proc: %v", err) + } + } else { + found = true + want6 = true + } + if found { + m.rxQueue.Set(uint64(rx + rx6)) + m.drops.Set(uint64(drops + drops6)) + } + case <-m.done: + t.Stop() + return + } + } +} + +// Registry returns the monitoring registry of the UDP metricset. +func (m *UDP) Registry() *monitoring.Registry { + if m == nil { + return nil + } + + return m.monitorRegistry +} + +// procNetUDP returns the rx_queue and drops field of the UDP socket table +// for the socket on the provided address formatted in hex, xxxxxxxx:xxxx or +// the IPv6 equivalent. +// This function is only useful on linux due to its dependence on the /proc +// filesystem, but is kept in this file for simplicity. If hasUnspecified +// is true, all addresses listed in the file in path are considered, and the +// sum of rx_queue and drops matching the addr ports is returned where the +// corresponding addrIsUnspecified is true. +func procNetUDP(path string, addr []string, hasUnspecified bool, addrIsUnspecified []bool) (rx, drops int64, err error) { + if len(addr) == 0 { + return 0, 0, nil + } + if len(addr) != len(addrIsUnspecified) { + return 0, 0, errors.New("mismatched address/unspecified lists: please report this") + } + b, err := os.ReadFile(path) + if err != nil { + return 0, 0, err + } + lines := bytes.Split(b, []byte("\n")) + if len(lines) < 2 { + return 0, 0, fmt.Errorf("%s entry not found for %s (no line)", path, addr) + } + var found bool + for _, l := range lines[1:] { + f := bytes.Fields(l) + const ( + queuesField = 4 + dropsField = 12 + ) + if len(f) > dropsField && contains(f[1], addr, addrIsUnspecified) { + _, r, ok := bytes.Cut(f[queuesField], []byte(":")) + if !ok { + return 0, 0, errors.New("no rx_queue field " + string(f[queuesField])) + } + found = true + + // queue lengths and drops are hex, e.g.: + // - https://elixir.bootlin.com/linux/v6.2.11/source/net/ipv4/udp.c#L3110 + // - https://elixir.bootlin.com/linux/v6.2.11/source/net/ipv6/datagram.c#L1048 + v, err := strconv.ParseInt(string(r), 16, 64) + if err != nil { + return 0, 0, fmt.Errorf("failed to parse rx_queue: %w", err) + } + rx += v + + v, err = strconv.ParseInt(string(f[dropsField]), 10, 64) + if err != nil { + return 0, 0, fmt.Errorf("failed to parse drops: %w", err) + } + drops += v + + if hasUnspecified { + continue + } + return rx, drops, nil + } + } + if found { + return rx, drops, nil + } + return 0, 0, fmt.Errorf("%s entry not found for %s", path, addr) +} + +// Close closes the UDP metricset and unregister the metrics. +func (m *UDP) Close() { + if m == nil { + return + } + if m.done != nil { + // Shut down poller and wait until done before unregistering metrics. + m.done <- struct{}{} + } + + if m.unregister != nil { + m.unregister() + m.unregister = nil + } + + m.monitorRegistry = nil +} diff --git a/filebeat/input/udp/input_test.go b/filebeat/input/netmetrics/udp_test.go similarity index 83% rename from filebeat/input/udp/input_test.go rename to filebeat/input/netmetrics/udp_test.go index 756f17375f90..6054c15ebc80 100644 --- a/filebeat/input/udp/input_test.go +++ b/filebeat/input/netmetrics/udp_test.go @@ -15,22 +15,20 @@ // specific language governing permissions and limitations // under the License. -package udp +package netmetrics import ( "net" "testing" "github.com/stretchr/testify/assert" - - "github.com/elastic/beats/v7/filebeat/input/internal/procnet" ) func TestProcNetUDP(t *testing.T) { t.Run("IPv4", func(t *testing.T) { path := "testdata/proc_net_udp.txt" t.Run("with_match", func(t *testing.T) { - addr := []string{procnet.IPv4(net.IP{0x0a, 0x64, 0x08, 0x25}, 0x1bbe)} + addr := []string{ipV4(net.IP{0x0a, 0x64, 0x08, 0x25}, 0x1bbe)} hasUnspecified, addrIsUnspecified, bad := containsUnspecifiedAddr(addr) rx, drops, err := procNetUDP(path, addr, hasUnspecified, addrIsUnspecified) if err != nil { @@ -42,7 +40,7 @@ func TestProcNetUDP(t *testing.T) { }) t.Run("leading_zero", func(t *testing.T) { - addr := []string{procnet.IPv4(net.IP{0x00, 0x7f, 0x01, 0x00}, 0x1eef)} + addr := []string{ipV4(net.IP{0x00, 0x7f, 0x01, 0x00}, 0x1eef)} hasUnspecified, addrIsUnspecified, bad := containsUnspecifiedAddr(addr) rx, drops, err := procNetUDP(path, addr, hasUnspecified, addrIsUnspecified) if err != nil { @@ -54,7 +52,7 @@ func TestProcNetUDP(t *testing.T) { }) t.Run("unspecified", func(t *testing.T) { - addr := []string{procnet.IPv4(net.ParseIP("0.0.0.0"), 0x1bbe)} + addr := []string{ipV4(net.ParseIP("0.0.0.0"), 0x1bbe)} hasUnspecified, addrIsUnspecified, bad := containsUnspecifiedAddr(addr) rx, drops, err := procNetUDP(path, addr, hasUnspecified, addrIsUnspecified) if err != nil { @@ -67,8 +65,8 @@ func TestProcNetUDP(t *testing.T) { t.Run("without_match", func(t *testing.T) { addr := []string{ - procnet.IPv4(net.IP{0xde, 0xad, 0xbe, 0xef}, 0xf00d), - procnet.IPv4(net.IP{0xba, 0x1d, 0xfa, 0xce}, 0x1135), + ipV4(net.IP{0xde, 0xad, 0xbe, 0xef}, 0xf00d), + ipV4(net.IP{0xba, 0x1d, 0xfa, 0xce}, 0x1135), } hasUnspecified, addrIsUnspecified, bad := containsUnspecifiedAddr(addr) _, _, err := procNetUDP(path, addr, hasUnspecified, addrIsUnspecified) @@ -92,7 +90,7 @@ func TestProcNetUDP(t *testing.T) { t.Run("IPv6", func(t *testing.T) { path := "testdata/proc_net_udp6.txt" t.Run("with_match", func(t *testing.T) { - addr := []string{procnet.IPv6(net.IP{0: 0x7f, 3: 0x01, 15: 0}, 0x1bbd)} + addr := []string{ipV6(net.IP{0: 0x7f, 3: 0x01, 15: 0}, 0x1bbd)} hasUnspecified, addrIsUnspecified, bad := containsUnspecifiedAddr(addr) rx, drops, err := procNetUDP(path, addr, hasUnspecified, addrIsUnspecified) if err != nil { @@ -104,7 +102,7 @@ func TestProcNetUDP(t *testing.T) { }) t.Run("leading_zero", func(t *testing.T) { - addr := []string{procnet.IPv6(net.IP{1: 0x7f, 2: 0x81, 15: 0}, 0x1eef)} + addr := []string{ipV6(net.IP{1: 0x7f, 2: 0x81, 15: 0}, 0x1eef)} hasUnspecified, addrIsUnspecified, bad := containsUnspecifiedAddr(addr) rx, drops, err := procNetUDP(path, addr, hasUnspecified, addrIsUnspecified) if err != nil { @@ -116,7 +114,7 @@ func TestProcNetUDP(t *testing.T) { }) t.Run("unspecified", func(t *testing.T) { - addr := []string{procnet.IPv6(net.ParseIP("[::]"), 0x1bbd)} + addr := []string{ipV6(net.ParseIP("[::]"), 0x1bbd)} hasUnspecified, addrIsUnspecified, bad := containsUnspecifiedAddr(addr) rx, drops, err := procNetUDP(path, addr, hasUnspecified, addrIsUnspecified) if err != nil { @@ -129,8 +127,8 @@ func TestProcNetUDP(t *testing.T) { t.Run("without_match", func(t *testing.T) { addr := []string{ - procnet.IPv6(net.IP{0xde, 0xad, 0xbe, 0xef, 0xde, 0xad, 0xbe, 0xef, 0xde, 0xad, 0xbe, 0xef, 0xde, 0xad, 0xbe, 0xef}, 0xf00d), - procnet.IPv6(net.IP{0xba, 0x1d, 0xfa, 0xce, 0xba, 0x1d, 0xfa, 0xce, 0xba, 0x1d, 0xfa, 0xce, 0xba, 0x1d, 0xfa, 0xce}, 0x1135), + ipV6(net.IP{0xde, 0xad, 0xbe, 0xef, 0xde, 0xad, 0xbe, 0xef, 0xde, 0xad, 0xbe, 0xef, 0xde, 0xad, 0xbe, 0xef}, 0xf00d), + ipV6(net.IP{0xba, 0x1d, 0xfa, 0xce, 0xba, 0x1d, 0xfa, 0xce, 0xba, 0x1d, 0xfa, 0xce, 0xba, 0x1d, 0xfa, 0xce}, 0x1135), } hasUnspecified, addrIsUnspecified, bad := containsUnspecifiedAddr(addr) _, _, err := procNetUDP(path, addr, hasUnspecified, addrIsUnspecified) diff --git a/filebeat/input/tcp/input.go b/filebeat/input/tcp/input.go index 1b3ffa7c2aa4..2502594b1aa2 100644 --- a/filebeat/input/tcp/input.go +++ b/filebeat/input/tcp/input.go @@ -18,21 +18,12 @@ package tcp import ( - "bytes" - "encoding/hex" - "errors" - "fmt" "net" - "os" - "runtime" - "strconv" - "strings" "time" "github.com/dustin/go-humanize" - "github.com/rcrowley/go-metrics" - "github.com/elastic/beats/v7/filebeat/input/internal/procnet" + "github.com/elastic/beats/v7/filebeat/input/netmetrics" input "github.com/elastic/beats/v7/filebeat/input/v2" stateless "github.com/elastic/beats/v7/filebeat/input/v2/input-stateless" "github.com/elastic/beats/v7/filebeat/inputsource" @@ -40,12 +31,9 @@ import ( "github.com/elastic/beats/v7/filebeat/inputsource/tcp" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/feature" - "github.com/elastic/beats/v7/libbeat/monitoring/inputmon" + conf "github.com/elastic/elastic-agent-libs/config" - "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" - "github.com/elastic/elastic-agent-libs/monitoring" - "github.com/elastic/elastic-agent-libs/monitoring/adapter" "github.com/elastic/go-concert/ctxtool" ) @@ -111,8 +99,8 @@ func (s *server) Run(ctx input.Context, publisher stateless.Publisher) error { defer log.Info("tcp input stopped") const pollInterval = time.Minute - metrics := newInputMetrics(ctx.ID, s.config.Host, pollInterval, log) - defer metrics.close() + metrics := netmetrics.NewTCP("tcp", ctx.ID, s.config.Host, pollInterval, log) + defer metrics.Close() split, err := streaming.SplitFunc(s.config.Framing, []byte(s.config.LineDelimiter)) if err != nil { @@ -139,7 +127,7 @@ func (s *server) Run(ctx input.Context, publisher stateless.Publisher) error { // This must be called after publisher.Publish to measure // the processing time metric. - metrics.log(data, evt.Timestamp) + metrics.Log(data, evt.Timestamp) }, split, )) @@ -156,235 +144,3 @@ func (s *server) Run(ctx input.Context, publisher stateless.Publisher) error { } return err } - -// inputMetrics handles the input's metric reporting. -type inputMetrics struct { - unregister func() - done chan struct{} - - lastPacket time.Time - - device *monitoring.String // name of the device being monitored - packets *monitoring.Uint // number of packets processed - bytes *monitoring.Uint // number of bytes processed - rxQueue *monitoring.Uint // value of the rx_queue field from /proc/net/tcp{,6} (only on linux systems) - arrivalPeriod metrics.Sample // histogram of the elapsed time between packet arrivals - processingTime metrics.Sample // histogram of the elapsed time between packet receipt and publication -} - -// newInputMetrics returns an input metric for the TCP processor. If id is empty -// a nil inputMetric is returned. -func newInputMetrics(id, device string, poll time.Duration, log *logp.Logger) *inputMetrics { - if id == "" { - return nil - } - reg, unreg := inputmon.NewInputRegistry("tcp", id, nil) - out := &inputMetrics{ - unregister: unreg, - device: monitoring.NewString(reg, "device"), - packets: monitoring.NewUint(reg, "received_events_total"), - bytes: monitoring.NewUint(reg, "received_bytes_total"), - rxQueue: monitoring.NewUint(reg, "receive_queue_length"), - arrivalPeriod: metrics.NewUniformSample(1024), - processingTime: metrics.NewUniformSample(1024), - } - _ = adapter.NewGoMetrics(reg, "arrival_period", adapter.Accept). - Register("histogram", metrics.NewHistogram(out.arrivalPeriod)) - _ = adapter.NewGoMetrics(reg, "processing_time", adapter.Accept). - Register("histogram", metrics.NewHistogram(out.processingTime)) - - out.device.Set(device) - - if poll > 0 && runtime.GOOS == "linux" { - addr4, addr6, err := procnet.Addrs(device, log) - if err != nil { - log.Warn(err) - return out - } - out.done = make(chan struct{}) - go out.poll(addr4, addr6, poll, log) - } - - return out -} - -// log logs metric for the given packet. -func (m *inputMetrics) log(data []byte, timestamp time.Time) { - if m == nil { - return - } - m.processingTime.Update(time.Since(timestamp).Nanoseconds()) - m.packets.Add(1) - m.bytes.Add(uint64(len(data))) - if !m.lastPacket.IsZero() { - m.arrivalPeriod.Update(timestamp.Sub(m.lastPacket).Nanoseconds()) - } - m.lastPacket = timestamp -} - -// poll periodically gets TCP buffer stats from the OS. -func (m *inputMetrics) poll(addr, addr6 []string, each time.Duration, log *logp.Logger) { - hasUnspecified, addrIsUnspecified, badAddr := containsUnspecifiedAddr(addr) - if badAddr != nil { - log.Warnf("failed to parse IPv4 addrs for metric collection %q", badAddr) - } - hasUnspecified6, addrIsUnspecified6, badAddr := containsUnspecifiedAddr(addr6) - if badAddr != nil { - log.Warnf("failed to parse IPv6 addrs for metric collection %q", badAddr) - } - - // Do an initial check for access to the filesystem and of the - // value constructed by containsUnspecifiedAddr. This gives a - // base level for the rx_queue values and ensures that if the - // constructed address values are malformed we panic early - // within the period of system testing. - want4 := true - rx, err := procNetTCP("/proc/net/tcp", addr, hasUnspecified, addrIsUnspecified) - if err != nil { - want4 = false - log.Infof("did not get initial tcp stats from /proc: %v", err) - } - want6 := true - rx6, err := procNetTCP("/proc/net/tcp6", addr6, hasUnspecified6, addrIsUnspecified6) - if err != nil { - want6 = false - log.Infof("did not get initial tcp6 stats from /proc: %v", err) - } - if !want4 && !want6 { - log.Warnf("failed to get initial tcp or tcp6 stats from /proc: %v", err) - } else { - m.rxQueue.Set(uint64(rx + rx6)) - } - - t := time.NewTicker(each) - for { - select { - case <-t.C: - var found bool - rx, err := procNetTCP("/proc/net/tcp", addr, hasUnspecified, addrIsUnspecified) - if err != nil { - if want4 { - log.Warnf("failed to get tcp stats from /proc: %v", err) - } - } else { - found = true - want4 = true - } - rx6, err := procNetTCP("/proc/net/tcp6", addr6, hasUnspecified6, addrIsUnspecified6) - if err != nil { - if want6 { - log.Warnf("failed to get tcp6 stats from /proc: %v", err) - } - } else { - found = true - want6 = true - } - if found { - m.rxQueue.Set(uint64(rx + rx6)) - } - case <-m.done: - t.Stop() - return - } - } -} - -func containsUnspecifiedAddr(addr []string) (yes bool, which []bool, bad []string) { - which = make([]bool, len(addr)) - for i, a := range addr { - prefix, _, ok := strings.Cut(a, ":") - if !ok { - continue - } - ip, err := hex.DecodeString(prefix) - if err != nil { - bad = append(bad, a) - } - if net.IP(ip).IsUnspecified() { - yes = true - which[i] = true - } - } - return yes, which, bad -} - -// procNetTCP returns the rx_queue field of the TCP socket table for the -// socket on the provided address formatted in hex, xxxxxxxx:xxxx or the IPv6 -// equivalent. -// This function is only useful on linux due to its dependence on the /proc -// filesystem, but is kept in this file for simplicity. If hasUnspecified -// is true, all addresses listed in the file in path are considered, and the -// sum of rx_queue matching the addr ports is returned where the corresponding -// addrIsUnspecified is true. -func procNetTCP(path string, addr []string, hasUnspecified bool, addrIsUnspecified []bool) (rx int64, err error) { - if len(addr) == 0 { - return 0, nil - } - if len(addr) != len(addrIsUnspecified) { - return 0, errors.New("mismatched address/unspecified lists: please report this") - } - b, err := os.ReadFile(path) - if err != nil { - return 0, err - } - lines := bytes.Split(b, []byte("\n")) - if len(lines) < 2 { - return 0, fmt.Errorf("%s entry not found for %s (no line)", path, addr) - } - var found bool - for _, l := range lines[1:] { - f := bytes.Fields(l) - const queuesField = 4 - if len(f) > queuesField && contains(f[1], addr, addrIsUnspecified) { - _, r, ok := bytes.Cut(f[4], []byte(":")) - if !ok { - return 0, errors.New("no rx_queue field " + string(f[queuesField])) - } - found = true - - // queue lengths are hex, e.g.: - // - https://elixir.bootlin.com/linux/v6.2.11/source/net/ipv4/tcp_ipv4.c#L2643 - // - https://elixir.bootlin.com/linux/v6.2.11/source/net/ipv6/tcp_ipv6.c#L1987 - v, err := strconv.ParseInt(string(r), 16, 64) - if err != nil { - return 0, fmt.Errorf("failed to parse rx_queue: %w", err) - } - rx += v - - if hasUnspecified { - continue - } - return rx, nil - } - } - if found { - return rx, nil - } - return 0, fmt.Errorf("%s entry not found for %s", path, addr) -} - -func contains(b []byte, addr []string, addrIsUnspecified []bool) bool { - for i, a := range addr { - if addrIsUnspecified[i] { - _, ap, pok := strings.Cut(a, ":") - _, bp, bok := bytes.Cut(b, []byte(":")) - if pok && bok && strings.EqualFold(string(bp), ap) { - return true - } - } else if strings.EqualFold(string(b), a) { - return true - } - } - return false -} - -func (m *inputMetrics) close() { - if m == nil { - return - } - if m.done != nil { - // Shut down poller and wait until done before unregistering metrics. - m.done <- struct{}{} - } - m.unregister() -} diff --git a/filebeat/input/udp/input.go b/filebeat/input/udp/input.go index cd7ca0c56051..190b77663ac4 100644 --- a/filebeat/input/udp/input.go +++ b/filebeat/input/udp/input.go @@ -18,33 +18,21 @@ package udp import ( - "bytes" - "encoding/hex" - "errors" - "fmt" "net" - "os" - "runtime" - "strconv" - "strings" "time" "github.com/dustin/go-humanize" - "github.com/rcrowley/go-metrics" - "github.com/elastic/beats/v7/filebeat/input/internal/procnet" + "github.com/elastic/beats/v7/filebeat/input/netmetrics" input "github.com/elastic/beats/v7/filebeat/input/v2" stateless "github.com/elastic/beats/v7/filebeat/input/v2/input-stateless" "github.com/elastic/beats/v7/filebeat/inputsource" "github.com/elastic/beats/v7/filebeat/inputsource/udp" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/feature" - "github.com/elastic/beats/v7/libbeat/monitoring/inputmon" + conf "github.com/elastic/elastic-agent-libs/config" - "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" - "github.com/elastic/elastic-agent-libs/monitoring" - "github.com/elastic/elastic-agent-libs/monitoring/adapter" "github.com/elastic/go-concert/ctxtool" ) @@ -107,8 +95,8 @@ func (s *server) Run(ctx input.Context, publisher stateless.Publisher) error { defer log.Info("udp input stopped") const pollInterval = time.Minute - metrics := newInputMetrics(ctx.ID, s.config.Host, uint64(s.config.ReadBuffer), pollInterval, log) - defer metrics.close() + metrics := netmetrics.NewUDP("udp", ctx.ID, s.config.Host, uint64(s.config.ReadBuffer), pollInterval, log) + defer metrics.Close() server := udp.New(&s.config.Config, func(data []byte, metadata inputsource.NetworkMetadata) { evt := beat.Event{ @@ -132,7 +120,7 @@ func (s *server) Run(ctx input.Context, publisher stateless.Publisher) error { // This must be called after publisher.Publish to measure // the processing time metric. - metrics.log(data, evt.Timestamp) + metrics.Log(data, evt.Timestamp) }) log.Debug("udp input initialized") @@ -144,251 +132,3 @@ func (s *server) Run(ctx input.Context, publisher stateless.Publisher) error { } return err } - -// inputMetrics handles the input's metric reporting. -type inputMetrics struct { - unregister func() - done chan struct{} - - lastPacket time.Time - - device *monitoring.String // name of the device being monitored - packets *monitoring.Uint // number of packets processed - bytes *monitoring.Uint // number of bytes processed - bufferLen *monitoring.Uint // configured read buffer length - rxQueue *monitoring.Uint // value of the rx_queue field from /proc/net/udp{,6} (only on linux systems) - drops *monitoring.Uint // number of udp drops noted in /proc/net/udp{,6} - arrivalPeriod metrics.Sample // histogram of the elapsed time between packet arrivals - processingTime metrics.Sample // histogram of the elapsed time between packet receipt and publication -} - -// newInputMetrics returns an input metric for the UDP processor. If id is empty -// a nil inputMetric is returned. -func newInputMetrics(id, device string, buflen uint64, poll time.Duration, log *logp.Logger) *inputMetrics { - if id == "" { - return nil - } - reg, unreg := inputmon.NewInputRegistry("udp", id, nil) - out := &inputMetrics{ - unregister: unreg, - bufferLen: monitoring.NewUint(reg, "udp_read_buffer_length_gauge"), - device: monitoring.NewString(reg, "device"), - packets: monitoring.NewUint(reg, "received_events_total"), - bytes: monitoring.NewUint(reg, "received_bytes_total"), - rxQueue: monitoring.NewUint(reg, "receive_queue_length"), - drops: monitoring.NewUint(reg, "system_packet_drops"), - arrivalPeriod: metrics.NewUniformSample(1024), - processingTime: metrics.NewUniformSample(1024), - } - _ = adapter.NewGoMetrics(reg, "arrival_period", adapter.Accept). - Register("histogram", metrics.NewHistogram(out.arrivalPeriod)) - _ = adapter.NewGoMetrics(reg, "processing_time", adapter.Accept). - Register("histogram", metrics.NewHistogram(out.processingTime)) - - out.device.Set(device) - out.bufferLen.Set(buflen) - - if poll > 0 && runtime.GOOS == "linux" { - addr, addr6, err := procnet.Addrs(device, log) - if err != nil { - log.Warn(err) - return out - } - out.done = make(chan struct{}) - go out.poll(addr, addr6, poll, log) - } - - return out -} - -// log logs metric for the given packet. -func (m *inputMetrics) log(data []byte, timestamp time.Time) { - if m == nil { - return - } - m.processingTime.Update(time.Since(timestamp).Nanoseconds()) - m.packets.Add(1) - m.bytes.Add(uint64(len(data))) - if !m.lastPacket.IsZero() { - m.arrivalPeriod.Update(timestamp.Sub(m.lastPacket).Nanoseconds()) - } - m.lastPacket = timestamp -} - -// poll periodically gets UDP buffer and packet drops stats from the OS. -func (m *inputMetrics) poll(addr, addr6 []string, each time.Duration, log *logp.Logger) { - hasUnspecified, addrIsUnspecified, badAddr := containsUnspecifiedAddr(addr) - if badAddr != nil { - log.Warnf("failed to parse IPv4 addrs for metric collection %q", badAddr) - } - hasUnspecified6, addrIsUnspecified6, badAddr := containsUnspecifiedAddr(addr6) - if badAddr != nil { - log.Warnf("failed to parse IPv6 addrs for metric collection %q", badAddr) - } - - // Do an initial check for access to the filesystem and of the - // value constructed by containsUnspecifiedAddr. This gives a - // base level for the rx_queue and drops values and ensures that - // if the constructed address values are malformed we panic early - // within the period of system testing. - want4 := true - rx, drops, err := procNetUDP("/proc/net/udp", addr, hasUnspecified, addrIsUnspecified) - if err != nil { - want4 = false - log.Infof("did not get initial udp stats from /proc: %v", err) - } - want6 := true - rx6, drops6, err := procNetUDP("/proc/net/udp6", addr6, hasUnspecified6, addrIsUnspecified6) - if err != nil { - want6 = false - log.Infof("did not get initial udp6 stats from /proc: %v", err) - } - if !want4 && !want6 { - log.Warnf("failed to get initial udp or udp6 stats from /proc: %v", err) - } else { - m.rxQueue.Set(uint64(rx + rx6)) - m.drops.Set(uint64(drops + drops6)) - } - - t := time.NewTicker(each) - for { - select { - case <-t.C: - var found bool - rx, drops, err := procNetUDP("/proc/net/udp", addr, hasUnspecified, addrIsUnspecified) - if err != nil { - if want4 { - log.Warnf("failed to get udp stats from /proc: %v", err) - } - } else { - found = true - want4 = true - } - rx6, drops6, err := procNetUDP("/proc/net/udp6", addr6, hasUnspecified6, addrIsUnspecified6) - if err != nil { - if want6 { - log.Warnf("failed to get udp6 stats from /proc: %v", err) - } - } else { - found = true - want6 = true - } - if found { - m.rxQueue.Set(uint64(rx + rx6)) - m.drops.Set(uint64(drops + drops6)) - } - case <-m.done: - t.Stop() - return - } - } -} - -func containsUnspecifiedAddr(addr []string) (yes bool, which []bool, bad []string) { - which = make([]bool, len(addr)) - for i, a := range addr { - prefix, _, ok := strings.Cut(a, ":") - if !ok { - continue - } - ip, err := hex.DecodeString(prefix) - if err != nil { - bad = append(bad, a) - } - if net.IP(ip).IsUnspecified() { - yes = true - which[i] = true - } - } - return yes, which, bad -} - -// procNetUDP returns the rx_queue and drops field of the UDP socket table -// for the socket on the provided address formatted in hex, xxxxxxxx:xxxx or -// the IPv6 equivalent. -// This function is only useful on linux due to its dependence on the /proc -// filesystem, but is kept in this file for simplicity. If hasUnspecified -// is true, all addresses listed in the file in path are considered, and the -// sum of rx_queue and drops matching the addr ports is returned where the -// corresponding addrIsUnspecified is true. -func procNetUDP(path string, addr []string, hasUnspecified bool, addrIsUnspecified []bool) (rx, drops int64, err error) { - if len(addr) == 0 { - return 0, 0, nil - } - if len(addr) != len(addrIsUnspecified) { - return 0, 0, errors.New("mismatched address/unspecified lists: please report this") - } - b, err := os.ReadFile(path) - if err != nil { - return 0, 0, err - } - lines := bytes.Split(b, []byte("\n")) - if len(lines) < 2 { - return 0, 0, fmt.Errorf("%s entry not found for %s (no line)", path, addr) - } - var found bool - for _, l := range lines[1:] { - f := bytes.Fields(l) - const ( - queuesField = 4 - dropsField = 12 - ) - if len(f) > dropsField && contains(f[1], addr, addrIsUnspecified) { - _, r, ok := bytes.Cut(f[queuesField], []byte(":")) - if !ok { - return 0, 0, errors.New("no rx_queue field " + string(f[queuesField])) - } - found = true - - // queue lengths and drops are hex, e.g.: - // - https://elixir.bootlin.com/linux/v6.2.11/source/net/ipv4/udp.c#L3110 - // - https://elixir.bootlin.com/linux/v6.2.11/source/net/ipv6/datagram.c#L1048 - v, err := strconv.ParseInt(string(r), 16, 64) - if err != nil { - return 0, 0, fmt.Errorf("failed to parse rx_queue: %w", err) - } - rx += v - - v, err = strconv.ParseInt(string(f[dropsField]), 10, 64) - if err != nil { - return 0, 0, fmt.Errorf("failed to parse drops: %w", err) - } - drops += v - - if hasUnspecified { - continue - } - return rx, drops, nil - } - } - if found { - return rx, drops, nil - } - return 0, 0, fmt.Errorf("%s entry not found for %s", path, addr) -} - -func contains(b []byte, addr []string, addrIsUnspecified []bool) bool { - for i, a := range addr { - if addrIsUnspecified[i] { - _, ap, pok := strings.Cut(a, ":") - _, bp, bok := bytes.Cut(b, []byte(":")) - if pok && bok && strings.EqualFold(string(bp), ap) { - return true - } - } else if strings.EqualFold(string(b), a) { - return true - } - } - return false -} - -func (m *inputMetrics) close() { - if m == nil { - return - } - if m.done != nil { - // Shut down poller and wait until done before unregistering metrics. - m.done <- struct{}{} - } - m.unregister() -} diff --git a/x-pack/filebeat/docs/inputs/input-netflow.asciidoc b/x-pack/filebeat/docs/inputs/input-netflow.asciidoc index 1b98f5f4395e..7d1023de148e 100644 --- a/x-pack/filebeat/docs/inputs/input-netflow.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-netflow.asciidoc @@ -145,4 +145,32 @@ which classifies RFC 1918 (IPv4) and RFC 4193 (IPv6) addresses as internal. [id="{beatname_lc}-input-{type}-common-options"] include::../../../../filebeat/docs/inputs/input-common-options.asciidoc[] +[float] +=== Metrics + +This input exposes metrics under the <>. +These metrics are exposed under the `/inputs/` path. They can be used to +observe the activity of the input. + +You must assign a unique `id` to the input to expose metrics. + +[options="header"] +|======= +| Metric | Description +| `device` | Host/port of the UDP stream. +| `udp_read_buffer_length_gauge` | Size of the UDP socket buffer length in bytes (gauge). +| `received_events_total` | Total number of packets (events) that have been received. +| `received_bytes_total` | Total number of bytes received. +| `receive_queue_length` | Aggregated size of the system receive queues (IPv4 and IPv6) (linux only) (gauge). +| `system_packet_drops` | Aggregated number of system packet drops (IPv4 and IPv6) (linux only) (gauge). +| `arrival_period` | Histogram of the time between successive packets in nanoseconds. +| `processing_time` | Histogram of the time taken to process packets in nanoseconds. +| `discarded_events_total` | Total number of discarded events. +| `decode_errors_total` | Total number of errors at decoding a packet. +| `flows_total` | Total number of received flows. +| `open_connections` | Number of current active netflow sessions. +|======= + +Histogram metrics are aggregated over the previous 1024 events. + :type!: diff --git a/x-pack/filebeat/input/netflow/decoder/config/config.go b/x-pack/filebeat/input/netflow/decoder/config/config.go index 3f750fd9fef5..5efe87e9630d 100644 --- a/x-pack/filebeat/input/netflow/decoder/config/config.go +++ b/x-pack/filebeat/input/netflow/decoder/config/config.go @@ -6,25 +6,30 @@ package config import ( "io" - "io/ioutil" "time" "github.com/elastic/beats/v7/x-pack/filebeat/input/netflow/decoder/fields" ) +type ActiveSessionsMetric interface { + Inc() + Dec() +} + // Config stores the configuration used by the NetFlow Collector. type Config struct { - protocols []string - logOutput io.Writer - expiration time.Duration - detectReset bool - fields fields.FieldDict - sharedTemplates bool + protocols []string + logOutput io.Writer + expiration time.Duration + detectReset bool + fields fields.FieldDict + sharedTemplates bool + activeSessionsMetric ActiveSessionsMetric } var defaultCfg = Config{ protocols: []string{}, - logOutput: ioutil.Discard, + logOutput: io.Discard, expiration: time.Hour, detectReset: true, sharedTemplates: false, @@ -91,6 +96,12 @@ func (c *Config) WithSharedTemplates(enabled bool) *Config { return c } +// WithActiveSessionsMetric configures the metric used to report active sessions. +func (c *Config) WithActiveSessionsMetric(metric ActiveSessionsMetric) *Config { + c.activeSessionsMetric = metric + return c +} + // Protocols returns a list of the protocols enabled. func (c *Config) Protocols() []string { return c.protocols @@ -119,3 +130,12 @@ func (c *Config) Fields() fields.FieldDict { } return c.fields } + +// ActiveSessionsMetric returns the configured metric to track active sessions. +func (c *Config) ActiveSessionsMetric() ActiveSessionsMetric { + if c == nil { + return nil + } + + return c.activeSessionsMetric +} diff --git a/x-pack/filebeat/input/netflow/decoder/ipfix/decoder.go b/x-pack/filebeat/input/netflow/decoder/ipfix/decoder.go index d236d5f23c71..fc4ab1e03d31 100644 --- a/x-pack/filebeat/input/netflow/decoder/ipfix/decoder.go +++ b/x-pack/filebeat/input/netflow/decoder/ipfix/decoder.go @@ -31,7 +31,7 @@ type DecoderIPFIX struct { var _ v9.Decoder = (*DecoderIPFIX)(nil) -func (_ DecoderIPFIX) ReadPacketHeader(buf *bytes.Buffer) (header v9.PacketHeader, newBuf *bytes.Buffer, countRecords int, err error) { +func (DecoderIPFIX) ReadPacketHeader(buf *bytes.Buffer) (header v9.PacketHeader, newBuf *bytes.Buffer, countRecords int, err error) { var data [SizeOfIPFIXHeader]byte n, err := buf.Read(data[:]) if n != len(data) || err != nil { diff --git a/x-pack/filebeat/input/netflow/decoder/v9/decoder.go b/x-pack/filebeat/input/netflow/decoder/v9/decoder.go index da82fbc12259..bd34b424d2f3 100644 --- a/x-pack/filebeat/input/netflow/decoder/v9/decoder.go +++ b/x-pack/filebeat/input/netflow/decoder/v9/decoder.go @@ -44,7 +44,7 @@ func (d DecoderV9) GetLogger() *log.Logger { return d.Logger } -func (_ DecoderV9) ReadPacketHeader(buf *bytes.Buffer) (header PacketHeader, newBuf *bytes.Buffer, numFlowSets int, err error) { +func (DecoderV9) ReadPacketHeader(buf *bytes.Buffer) (header PacketHeader, newBuf *bytes.Buffer, numFlowSets int, err error) { var data [20]byte n, err := buf.Read(data[:]) if n != len(data) || err != nil { @@ -61,7 +61,7 @@ func (_ DecoderV9) ReadPacketHeader(buf *bytes.Buffer) (header PacketHeader, new return header, buf, int(header.Count), nil } -func (_ DecoderV9) ReadSetHeader(buf *bytes.Buffer) (SetHeader, error) { +func (DecoderV9) ReadSetHeader(buf *bytes.Buffer) (SetHeader, error) { var data [4]byte n, err := buf.Read(data[:]) if n != len(data) || err != nil { @@ -181,7 +181,7 @@ func (d DecoderV9) ReadOptionsTemplateFlowSet(buf *bytes.Buffer) (templates []*t scopeLen := int(binary.BigEndian.Uint16(header[2:4])) optsLen := int(binary.BigEndian.Uint16(header[4:])) length := optsLen + scopeLen - if buf.Len() < int(length) { + if buf.Len() < length { return nil, io.EOF } if (scopeLen+optsLen) == 0 || scopeLen&3 != 0 || optsLen&3 != 0 { diff --git a/x-pack/filebeat/input/netflow/decoder/v9/session.go b/x-pack/filebeat/input/netflow/decoder/v9/session.go index 0baaa8e671ca..492576f6b962 100644 --- a/x-pack/filebeat/input/netflow/decoder/v9/session.go +++ b/x-pack/filebeat/input/netflow/decoder/v9/session.go @@ -11,6 +11,7 @@ import ( "time" "github.com/elastic/beats/v7/x-pack/filebeat/input/netflow/decoder/atomic" + "github.com/elastic/beats/v7/x-pack/filebeat/input/netflow/decoder/config" "github.com/elastic/beats/v7/x-pack/filebeat/input/netflow/decoder/template" ) @@ -113,7 +114,7 @@ func (s *SessionState) CheckReset(seqNum uint32) (prev uint32, reset bool) { s.Templates = make(map[TemplateKey]*TemplateWrapper) } s.lastSequence = seqNum - return + return prev, reset } func isValidSequence(current, next uint32) bool { @@ -125,16 +126,34 @@ type SessionMap struct { mutex sync.RWMutex Sessions map[SessionKey]*SessionState logger *log.Logger + metric config.ActiveSessionsMetric } // NewSessionMap returns a new SessionMap. -func NewSessionMap(logger *log.Logger) SessionMap { +func NewSessionMap(logger *log.Logger, metric config.ActiveSessionsMetric) SessionMap { return SessionMap{ logger: logger, Sessions: make(map[SessionKey]*SessionState), + metric: metric, } } +func (m *SessionMap) decreaseActiveSessions() { + if m.metric == nil { + return + } + + m.metric.Dec() +} + +func (m *SessionMap) increaseActiveSessions() { + if m.metric == nil { + return + } + + m.metric.Inc() +} + // GetOrCreate looks up the given session key and returns an existing session // or creates a new one. func (m *SessionMap) GetOrCreate(key SessionKey) *SessionState { @@ -149,6 +168,7 @@ func (m *SessionMap) GetOrCreate(key SessionKey) *SessionState { if session, found = m.Sessions[key]; !found { session = NewSession(m.logger) m.Sessions[key] = session + m.increaseActiveSessions() } m.mutex.Unlock() } @@ -175,6 +195,7 @@ func (m *SessionMap) cleanup() (aliveSession int, removedSession int, aliveTempl if session, found := m.Sessions[key]; found && session.Delete.Load() { delete(m.Sessions, key) removedSession++ + m.decreaseActiveSessions() } } m.mutex.Unlock() diff --git a/x-pack/filebeat/input/netflow/decoder/v9/session_test.go b/x-pack/filebeat/input/netflow/decoder/v9/session_test.go index 80c91845c0a6..8c10b2b98e94 100644 --- a/x-pack/filebeat/input/netflow/decoder/v9/session_test.go +++ b/x-pack/filebeat/input/netflow/decoder/v9/session_test.go @@ -5,7 +5,7 @@ package v9 import ( - "io/ioutil" + "io" "log" "math" "sync" @@ -18,7 +18,7 @@ import ( "github.com/elastic/beats/v7/x-pack/filebeat/input/netflow/decoder/test" ) -var logger = log.New(ioutil.Discard, "", 0) +var logger = log.New(io.Discard, "", 0) func makeSessionKey(t testing.TB, ipPortPair string, domain uint32) SessionKey { return MakeSessionKey(test.MakeAddress(t, ipPortPair), domain, false) @@ -26,7 +26,7 @@ func makeSessionKey(t testing.TB, ipPortPair string, domain uint32) SessionKey { func TestSessionMap_GetOrCreate(t *testing.T) { t.Run("consistent behavior", func(t *testing.T) { - sm := NewSessionMap(logger) + sm := NewSessionMap(logger, nil) // Session is created s1 := sm.GetOrCreate(makeSessionKey(t, "127.0.0.1:1234", 42)) @@ -59,7 +59,7 @@ func TestSessionMap_GetOrCreate(t *testing.T) { }) t.Run("parallel", func(t *testing.T) { // Goroutines should observe the same session when created in parallel - sm := NewSessionMap(logger) + sm := NewSessionMap(logger, nil) key := makeSessionKey(t, "127.0.0.1:9995", 42) const N = 8 const Iters = 200 @@ -101,7 +101,7 @@ func testTemplate(id uint16) *template.Template { } func TestSessionState(t *testing.T) { - logger := log.New(ioutil.Discard, "", 0) + logger := log.New(io.Discard, "", 0) t.Run("create and get", func(t *testing.T) { s := NewSession(logger) t1 := testTemplate(1) @@ -128,12 +128,12 @@ func TestSessionState(t *testing.T) { t1c = s.GetTemplate(1) assert.False(t, t1 == t1c) - assert.True(t, t1b == t1b) + assert.True(t, t1b == t1c) }) } func TestSessionMap_Cleanup(t *testing.T) { - sm := NewSessionMap(logger) + sm := NewSessionMap(logger, nil) // Session is created k1 := makeSessionKey(t, "127.0.0.1:1234", 1) @@ -180,7 +180,7 @@ func TestSessionMap_Cleanup(t *testing.T) { func TestSessionMap_CleanupLoop(t *testing.T) { timeout := time.Millisecond * 100 - sm := NewSessionMap(log.New(ioutil.Discard, "", 0)) + sm := NewSessionMap(log.New(io.Discard, "", 0), nil) key := makeSessionKey(t, "127.0.0.1:1", 42) s := sm.GetOrCreate(key) diff --git a/x-pack/filebeat/input/netflow/decoder/v9/v9.go b/x-pack/filebeat/input/netflow/decoder/v9/v9.go index fdb46076c875..7f17a24f4faf 100644 --- a/x-pack/filebeat/input/netflow/decoder/v9/v9.go +++ b/x-pack/filebeat/input/netflow/decoder/v9/v9.go @@ -34,7 +34,7 @@ type NetflowV9Protocol struct { } func init() { - protocol.Registry.Register(ProtocolName, New) + _ = protocol.Registry.Register(ProtocolName, New) } func New(config config.Config) protocol.Protocol { @@ -45,7 +45,7 @@ func New(config config.Config) protocol.Protocol { func NewProtocolWithDecoder(decoder Decoder, config config.Config, logger *log.Logger) *NetflowV9Protocol { return &NetflowV9Protocol{ decoder: decoder, - Session: NewSessionMap(logger), + Session: NewSessionMap(logger, config.ActiveSessionsMetric()), logger: logger, timeout: config.ExpirationTimeout(), detectReset: config.SequenceResetEnabled(), diff --git a/x-pack/filebeat/input/netflow/input.go b/x-pack/filebeat/input/netflow/input.go index addd3d39c25d..a87fe6a0d76b 100644 --- a/x-pack/filebeat/input/netflow/input.go +++ b/x-pack/filebeat/input/netflow/input.go @@ -6,25 +6,22 @@ package netflow import ( "bytes" - "context" "fmt" "net" "sync" "time" + "github.com/elastic/beats/v7/filebeat/input/netmetrics" v2 "github.com/elastic/beats/v7/filebeat/input/v2" "github.com/elastic/beats/v7/filebeat/inputsource" "github.com/elastic/beats/v7/filebeat/inputsource/udp" "github.com/elastic/beats/v7/libbeat/beat" - "github.com/elastic/beats/v7/libbeat/common/atomic" "github.com/elastic/beats/v7/libbeat/feature" "github.com/elastic/beats/v7/x-pack/filebeat/input/netflow/decoder" "github.com/elastic/beats/v7/x-pack/filebeat/input/netflow/decoder/fields" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" - "github.com/elastic/elastic-agent-libs/monitoring" - "github.com/elastic/go-concert/ctxtool" "github.com/elastic/go-concert/unison" ) @@ -32,13 +29,6 @@ const ( inputName = "netflow" ) -var ( - numPackets = monitoring.NewUint(nil, "filebeat.input.netflow.packets.received") - numDropped = monitoring.NewUint(nil, "filebeat.input.netflow.packets.dropped") - numFlows = monitoring.NewUint(nil, "filebeat.input.netflow.flows") - aliveInputs atomic.Int -) - func Plugin(log *logp.Logger) v2.Plugin { return v2.Plugin{ Name: inputName, @@ -74,26 +64,13 @@ func (im *netflowInputManager) Create(cfg *conf.C) (v2.Input, error) { customFields[idx] = f } - dec, err := decoder.NewDecoder(decoder.NewConfig(). - WithProtocols(inputCfg.Protocols...). - WithExpiration(inputCfg.ExpirationTimeout). - WithLogOutput(&logDebugWrapper{Logger: im.log}). - WithCustomFields(customFields...). - WithSequenceResetEnabled(inputCfg.DetectSequenceReset). - WithSharedTemplates(inputCfg.ShareTemplates)) - if err != nil { - return nil, fmt.Errorf("error initializing netflow decoder: %w", err) - } - input := &netflowInput{ - decoder: dec, + cfg: inputCfg, internalNetworks: inputCfg.InternalNetworks, logger: im.log, queueSize: inputCfg.PacketQueueSize, } - input.udp = udp.New(&inputCfg.Config, input.packetDispatch) - return input, nil } @@ -104,9 +81,10 @@ type packet struct { type netflowInput struct { mtx sync.Mutex - udp *udp.Server + cfg config decoder *decoder.Decoder client beat.Client + customFields []fields.FieldDict internalNetworks []string logger *logp.Logger queueC chan packet @@ -122,16 +100,7 @@ func (n *netflowInput) Test(_ v2.TestContext) error { return nil } -func (n *netflowInput) packetDispatch(data []byte, metadata inputsource.NetworkMetadata) { - select { - case n.queueC <- packet{data, metadata.RemoteAddr}: - numPackets.Inc() - default: - numDropped.Inc() - } -} - -func (n *netflowInput) Run(context v2.Context, connector beat.PipelineConnector) error { +func (n *netflowInput) Run(ctx v2.Context, connector beat.PipelineConnector) error { n.mtx.Lock() if n.started { n.mtx.Unlock() @@ -151,7 +120,7 @@ func (n *netflowInput) Run(context v2.Context, connector beat.PipelineConnector) // is not required. EventNormalization: boolPtr(false), }, - CloseRef: context.Cancelation, + CloseRef: ctx.Cancelation, EventListener: nil, }) if err != nil { @@ -160,6 +129,24 @@ func (n *netflowInput) Run(context v2.Context, connector beat.PipelineConnector) return err } + const pollInterval = time.Minute + udpMetrics := netmetrics.NewUDP("netflow", ctx.ID, n.cfg.Host, uint64(n.cfg.ReadBuffer), pollInterval, n.logger) + defer udpMetrics.Close() + + flowMetrics := newInputMetrics(udpMetrics.Registry()) + + n.decoder, err = decoder.NewDecoder(decoder.NewConfig(). + WithProtocols(n.cfg.Protocols...). + WithExpiration(n.cfg.ExpirationTimeout). + WithLogOutput(&logDebugWrapper{Logger: n.logger}). + WithCustomFields(n.customFields...). + WithSequenceResetEnabled(n.cfg.DetectSequenceReset). + WithSharedTemplates(n.cfg.ShareTemplates). + WithActiveSessionsMetric(flowMetrics.ActiveSessions())) + if err != nil { + return fmt.Errorf("error initializing netflow decoder: %w", err) + } + n.logger.Info("Starting netflow decoder") if err := n.decoder.Start(); err != nil { n.logger.Errorw("Failed to start netflow decoder", "error", err) @@ -170,20 +157,26 @@ func (n *netflowInput) Run(context v2.Context, connector beat.PipelineConnector) n.queueC = make(chan packet, n.queueSize) n.logger.Info("Starting udp server") - err = n.udp.Start() + + udpServer := udp.New(&n.cfg.Config, func(data []byte, metadata inputsource.NetworkMetadata) { + select { + case n.queueC <- packet{data, metadata.RemoteAddr}: + default: + if discardedEvents := flowMetrics.DiscardedEvents(); discardedEvents != nil { + discardedEvents.Inc() + } + } + }) + err = udpServer.Start() if err != nil { n.logger.Errorf("Failed to start udp server: %v", err) n.stop() return err } - - if aliveInputs.Inc() == 1 && n.logger.IsDebug() { - go n.statsLoop(ctxtool.FromCanceller(context.Cancelation)) - } - defer aliveInputs.Dec() + defer udpServer.Stop() go func() { - <-context.Cancelation.Done() + <-ctx.Cancelation.Done() n.stop() }() @@ -191,6 +184,9 @@ func (n *netflowInput) Run(context v2.Context, connector beat.PipelineConnector) flows, err := n.decoder.Read(bytes.NewBuffer(packet.data), packet.source) if err != nil { n.logger.Warnf("Error parsing NetFlow packet of length %d from %s: %v", len(packet.data), packet.source, err) + if decodeErrors := flowMetrics.DecodeErrors(); decodeErrors != nil { + decodeErrors.Inc() + } continue } @@ -199,11 +195,19 @@ func (n *netflowInput) Run(context v2.Context, connector beat.PipelineConnector) continue } evs := make([]beat.Event, fLen) - numFlows.Add(uint64(fLen)) + if flowsTotal := flowMetrics.Flows(); flowsTotal != nil { + flowsTotal.Add(uint64(fLen)) + } for i, flow := range flows { evs[i] = toBeatEvent(flow, n.internalNetworks) } client.PublishAll(evs) + + // This must be called after publisher.PublishAll to measure + // the processing time metric. also we pass time.Now() as we have + // multiple flows resulting in multiple events of which the timestamp + // is obtained from the NetFlow header + udpMetrics.Log(packet.data, time.Now()) } return nil @@ -238,20 +242,18 @@ func (n *netflowInput) stop() { return } - if n.udp != nil { - n.udp.Stop() - } - if n.decoder != nil { if err := n.decoder.Stop(); err != nil { n.logger.Errorw("Error stopping decoder", "error", err) } + n.decoder = nil } if n.client != nil { if err := n.client.Close(); err != nil { n.logger.Errorw("Error closing beat client", "error", err) } + n.client = nil } close(n.queueC) @@ -259,42 +261,4 @@ func (n *netflowInput) stop() { n.started = false } -func (n *netflowInput) statsLoop(ctx context.Context) { - prevPackets := numPackets.Get() - prevFlows := numFlows.Get() - prevDropped := numDropped.Get() - // The stats thread only monitors queue length for the first input - prevQueue := len(n.queueC) - t := time.NewTicker(time.Second) - defer t.Stop() - for { - select { - case <-t.C: - packets := numPackets.Get() - flows := numFlows.Get() - dropped := numDropped.Get() - queue := len(n.queueC) - if packets > prevPackets || flows > prevFlows || dropped > prevDropped || queue > prevQueue { - n.logger.Debugf("Stats total:[ packets=%d dropped=%d flows=%d queue_len=%d ] delta:[ packets/s=%d dropped/s=%d flows/s=%d queue_len/s=%+d ]", - packets, dropped, flows, queue, packets-prevPackets, dropped-prevDropped, flows-prevFlows, queue-prevQueue) - prevFlows = flows - prevPackets = packets - prevQueue = queue - prevDropped = dropped - continue - } - - n.mtx.Lock() - count := aliveInputs.Load() - n.mtx.Unlock() - if count == 0 { - return - } - - case <-ctx.Done(): - return - } - } -} - func boolPtr(b bool) *bool { return &b } diff --git a/x-pack/filebeat/input/netflow/metrics.go b/x-pack/filebeat/input/netflow/metrics.go new file mode 100644 index 000000000000..20fbede23b76 --- /dev/null +++ b/x-pack/filebeat/input/netflow/metrics.go @@ -0,0 +1,55 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package netflow + +import "github.com/elastic/elastic-agent-libs/monitoring" + +type netflowMetrics struct { + discardedEvents *monitoring.Uint + decodeErrors *monitoring.Uint + flows *monitoring.Uint + activeSessions *monitoring.Uint +} + +func newInputMetrics(reg *monitoring.Registry) *netflowMetrics { + if reg == nil { + return nil + } + + return &netflowMetrics{ + discardedEvents: monitoring.NewUint(reg, "discarded_events_total"), + flows: monitoring.NewUint(reg, "flows_total"), + decodeErrors: monitoring.NewUint(reg, "decode_errors_total"), + activeSessions: monitoring.NewUint(reg, "open_connections"), + } +} + +func (n *netflowMetrics) DiscardedEvents() *monitoring.Uint { + if n == nil { + return nil + } + return n.discardedEvents +} + +func (n *netflowMetrics) DecodeErrors() *monitoring.Uint { + if n == nil { + return nil + } + return n.decodeErrors +} + +func (n *netflowMetrics) Flows() *monitoring.Uint { + if n == nil { + return nil + } + return n.flows +} + +func (n *netflowMetrics) ActiveSessions() *monitoring.Uint { + if n == nil { + return nil + } + return n.activeSessions +} From 6dc03053c4eb4d055db089acc5ae261ec3e87280 Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Thu, 22 Feb 2024 13:43:12 -0500 Subject: [PATCH 11/26] chore: Update snapshot.yml (#38102) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Made with ❤️️ by updatecli Co-authored-by: apmmachine --- testing/environments/snapshot.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index 4029dd7fa978..fd3c6007409e 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.13.0-772867d3-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.14.0-74a79bf3-SNAPSHOT # When extend is used it merges healthcheck.tests, see: # https://github.com/docker/compose/issues/8962 # healthcheck: @@ -31,7 +31,7 @@ services: - "./docker/elasticsearch/users_roles:/usr/share/elasticsearch/config/users_roles" logstash: - image: docker.elastic.co/logstash/logstash:8.13.0-772867d3-SNAPSHOT + image: docker.elastic.co/logstash/logstash:8.14.0-74a79bf3-SNAPSHOT healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9600/_node/stats"] retries: 600 @@ -44,7 +44,7 @@ services: - 5055:5055 kibana: - image: docker.elastic.co/kibana/kibana:8.13.0-772867d3-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.14.0-74a79bf3-SNAPSHOT environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing" From a27e4399dfada7bfd30acc1a6d96f3683c3e4d28 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 22 Feb 2024 19:29:41 -0500 Subject: [PATCH 12/26] build(deps): bump cryptography in /libbeat/tests/system (#38108) Bumps [cryptography](https://github.com/pyca/cryptography) from 41.0.7 to 42.0.4. - [Changelog](https://github.com/pyca/cryptography/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pyca/cryptography/compare/41.0.7...42.0.4) --- updated-dependencies: - dependency-name: cryptography dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- libbeat/tests/system/requirements.txt | 2 +- libbeat/tests/system/requirements_aix.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libbeat/tests/system/requirements.txt b/libbeat/tests/system/requirements.txt index 8bdb021e8ec5..fc4227738c32 100644 --- a/libbeat/tests/system/requirements.txt +++ b/libbeat/tests/system/requirements.txt @@ -9,7 +9,7 @@ certifi==2023.7.22 cffi==1.16.0 chardet==3.0.4 charset-normalizer==3.3.2 -cryptography==41.0.7 +cryptography==42.0.4 deepdiff==4.2.0 Deprecated==1.2.14 distro==1.9.0 diff --git a/libbeat/tests/system/requirements_aix.txt b/libbeat/tests/system/requirements_aix.txt index 8bdb021e8ec5..fc4227738c32 100644 --- a/libbeat/tests/system/requirements_aix.txt +++ b/libbeat/tests/system/requirements_aix.txt @@ -9,7 +9,7 @@ certifi==2023.7.22 cffi==1.16.0 chardet==3.0.4 charset-normalizer==3.3.2 -cryptography==41.0.7 +cryptography==42.0.4 deepdiff==4.2.0 Deprecated==1.2.14 distro==1.9.0 From 353dab322cd6c86fdd082a96b862f0993dac0034 Mon Sep 17 00:00:00 2001 From: Dan Kortschak <90160302+efd6@users.noreply.github.com> Date: Sat, 24 Feb 2024 06:08:39 +1030 Subject: [PATCH 13/26] x-pack/filebeat/input/httpjson: drop response bodies at end of execution (#38116) The response bodies of the first and last responses were being held in a closed-over variable resulting in high static memory loads in some situations. The bodies are not used between periodic executions with the documentation stating that only cursor values are persisted across restarts. The difference in behaviour between using the body field over a restart versus over a sequence of executions in the same run make them unsafe, so clarify the persistence behaviour in the documentation and free the bodies at the end of an execution. A survey of integrations that use the httpjson input did not identify any that are using behaviour that is being removed, but we will need to keep an eye on cases that may have been missed. In general, if persistence is being depended on, the cursor should be being used. --- CHANGELOG.next.asciidoc | 1 + x-pack/filebeat/docs/inputs/input-httpjson.asciidoc | 2 +- x-pack/filebeat/input/httpjson/input.go | 6 ++++++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index d7761c359dca..46a86a51ecdc 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -96,6 +96,7 @@ fields added to events containing the Beats version. {pull}37553[37553] - Update github.com/lestrrat-go/jwx dependency. {pull}37799[37799] - [threatintel] MISP pagination fixes {pull}37898[37898] - Fix file handle leak when handling errors in filestream {pull}37973[37973] +- Prevent HTTPJSON holding response bodies between executions. {issue}35219[35219] {pull}38116[38116] *Heartbeat* diff --git a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc index 5fbd5dc15a5f..bf2b9f195cc2 100644 --- a/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-httpjson.asciidoc @@ -119,7 +119,7 @@ The state has the following elements: - `body`: A map containing the body. References the next request body when used in <> or <> configuration sections, and to the last response body when used in <> or <> configuration sections. - `cursor`: A map containing any data the user configured to be stored between restarts (See <>). -All of the mentioned objects are only stored at runtime, except `cursor`, which has values that are persisted between restarts. +All of the mentioned objects are only stored at runtime during the execution of the periodic request, except `cursor`, which has values that are persisted between periodic request and restarts. [[transforms]] ==== Transforms diff --git a/x-pack/filebeat/input/httpjson/input.go b/x-pack/filebeat/input/httpjson/input.go index 50a4f7f20a61..6757883a8a12 100644 --- a/x-pack/filebeat/input/httpjson/input.go +++ b/x-pack/filebeat/input/httpjson/input.go @@ -163,6 +163,12 @@ func run(ctx v2.Context, cfg config, pub inputcursor.Publisher, crsr *inputcurso trCtx.cursor.load(crsr) doFunc := func() error { + defer func() { + // Clear response bodies between evaluations. + trCtx.firstResponse.body = nil + trCtx.lastResponse.body = nil + }() + log.Info("Process another repeated request.") startTime := time.Now() From ef7115ca63138437da83c734008d63c6d54e5724 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Sat, 24 Feb 2024 08:13:39 -0500 Subject: [PATCH 14/26] Add Agent diagnostic hook to dump beat metrics (#38115) Register an Agent diagnostic hook that will dump all metrics registered into the default monitoring namespace as well as expvar. The hook is registered within libbeat so this change will affect all Beats operating under Agent. The file will be named beat_metrics.json and will contain a single pretty JSON object. Closes #37929 --- libbeat/cmd/instance/beat.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go index 4b7470b1dbd5..f25a24d2d5aa 100644 --- a/libbeat/cmd/instance/beat.go +++ b/libbeat/cmd/instance/beat.go @@ -893,6 +893,16 @@ func (b *Beat) configure(settings Settings) error { b.Manager.RegisterDiagnosticHook("global processors", "a list of currently configured global beat processors", "global_processors.txt", "text/plain", b.agentDiagnosticHook) + b.Manager.RegisterDiagnosticHook("beat_metrics", "Metrics from the default monitoring namespace and expvar.", + "beat_metrics.json", "application/json", func() []byte { + m := monitoring.CollectStructSnapshot(monitoring.Default, monitoring.Full, true) + data, err := json.MarshalIndent(m, "", " ") + if err != nil { + logp.L().Warnw("Failed to collect beat metric snapshot for Agent diagnostics.", "error", err) + return []byte(err.Error()) + } + return data + }) return err } From 0361e30247f8f22ad7fb9d85858180e6ee8d3c16 Mon Sep 17 00:00:00 2001 From: sharbuz <87968844+sharbuz@users.noreply.github.com> Date: Mon, 26 Feb 2024 13:16:16 +0200 Subject: [PATCH 15/26] increase the maximum timeout for x-pack/metricbeat pipeline (#38131) --- catalog-info.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalog-info.yaml b/catalog-info.yaml index 15fc06f85c9b..955c316d5aae 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -637,7 +637,7 @@ spec: spec: # branch_configuration: "7.17" #TODO: uncomment after tests pipeline_file: ".buildkite/x-pack/pipeline.xpack.metricbeat.yml" - maximum_timeout_in_minutes: 120 + maximum_timeout_in_minutes: 240 provider_settings: trigger_mode: none # don't trigger jobs from github activity build_pull_request_forks: false From 27cde877d9cadc92ddf32113a266e4c5bcab62f8 Mon Sep 17 00:00:00 2001 From: Alberto Delgado Roda Date: Mon, 4 Mar 2024 11:09:50 +0100 Subject: [PATCH 16/26] [Heartbeat] Adjust State loader to only retry for failed requests and not for 4xx (#37981) * only retry when the status is 5xx * remove test AAA comments * add changelog * correct changelog modification * fix ES query * change error handling strategy * do not retry when there is malformed data * improve retry mechanism * improve log message * improve changelog * fix log format --- CHANGELOG.next.asciidoc | 3 + .../wrappers/monitorstate/esloader.go | 22 +++++-- .../wrappers/monitorstate/esloader_test.go | 63 ++++++++++++++++++- .../wrappers/monitorstate/testutil.go | 4 +- .../monitors/wrappers/monitorstate/tracker.go | 34 +++++++--- .../wrappers/monitorstate/tracker_test.go | 49 +++++++++++++++ 6 files changed, 160 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 46a86a51ecdc..eef3e2304167 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -100,6 +100,9 @@ fields added to events containing the Beats version. {pull}37553[37553] *Heartbeat* +- Fix panics when parsing dereferencing invalid parsed url. {pull}34702[34702] +- Fix setuid root when running under cgroups v2. {pull}37794[37794] +- Adjust State loader to only retry when response code status is 5xx {pull}37981[37981] *Metricbeat* diff --git a/heartbeat/monitors/wrappers/monitorstate/esloader.go b/heartbeat/monitors/wrappers/monitorstate/esloader.go index d63be5aada5b..5705ec4b1468 100644 --- a/heartbeat/monitors/wrappers/monitorstate/esloader.go +++ b/heartbeat/monitors/wrappers/monitorstate/esloader.go @@ -32,13 +32,21 @@ import ( var DefaultDataStreams = "synthetics-*,heartbeat-*" +type LoaderError struct { + err error + Retry bool +} + +func (e LoaderError) Error() string { + return e.err.Error() +} + func MakeESLoader(esc *eslegclient.Connection, indexPattern string, beatLocation *config.LocationWithID) StateLoader { if indexPattern == "" { // Should never happen, but if we ever make a coding error... logp.L().Warn("ES state loader initialized with no index pattern, will not load states from ES") return NilStateLoader } - return func(sf stdfields.StdMonitorFields) (*State, error) { var runFromID string if sf.RunFrom != nil { @@ -74,10 +82,11 @@ func MakeESLoader(esc *eslegclient.Connection, indexPattern string, beatLocation }, }, } - status, body, err := esc.Request("POST", strings.Join([]string{"/", indexPattern, "/", "_search", "?size=1"}, ""), "", nil, reqBody) if err != nil || status > 299 { - return nil, fmt.Errorf("error executing state search for %s in loc=%s: %w", sf.ID, runFromID, err) + sErr := fmt.Errorf("error executing state search for %s in loc=%s: %w", sf.ID, runFromID, err) + retry := shouldRetry(status) + return nil, LoaderError{err: sErr, Retry: retry} } type stateHits struct { @@ -94,7 +103,8 @@ func MakeESLoader(esc *eslegclient.Connection, indexPattern string, beatLocation sh := stateHits{} err = json.Unmarshal(body, &sh) if err != nil { - return nil, fmt.Errorf("could not unmarshal state hits for %s: %w", sf.ID, err) + sErr := fmt.Errorf("could not unmarshal state hits for %s: %w", sf.ID, err) + return nil, LoaderError{err: sErr, Retry: false} } if len(sh.Hits.Hits) == 0 { @@ -107,3 +117,7 @@ func MakeESLoader(esc *eslegclient.Connection, indexPattern string, beatLocation return state, nil } } + +func shouldRetry(status int) bool { + return status >= 500 +} diff --git a/heartbeat/monitors/wrappers/monitorstate/esloader_test.go b/heartbeat/monitors/wrappers/monitorstate/esloader_test.go index 42b1c6c31c31..db70c1cc72e7 100644 --- a/heartbeat/monitors/wrappers/monitorstate/esloader_test.go +++ b/heartbeat/monitors/wrappers/monitorstate/esloader_test.go @@ -21,6 +21,9 @@ package monitorstate import ( "fmt" + "io" + "net/http" + "strings" "testing" "time" @@ -33,6 +36,7 @@ import ( "github.com/elastic/beats/v7/heartbeat/config" "github.com/elastic/beats/v7/heartbeat/esutil" "github.com/elastic/beats/v7/heartbeat/monitors/stdfields" + "github.com/elastic/beats/v7/libbeat/esleg/eslegclient" "github.com/elastic/beats/v7/libbeat/processors/util" ) @@ -51,7 +55,7 @@ func TestStatesESLoader(t *testing.T) { monID := etc.createTestMonitorStateInES(t, testStatus) // Since we've continued this state it should register the initial state - ms := etc.tracker.GetCurrentState(monID) + ms := etc.tracker.GetCurrentState(monID, RetryConfig{}) require.True(t, ms.StartedAt.After(testStart.Add(-time.Nanosecond)), "timestamp for new state is off") requireMSStatusCount(t, ms, testStatus, 1) @@ -89,8 +93,61 @@ func TestStatesESLoader(t *testing.T) { } } +func TestMakeESLoaderError(t *testing.T) { + tests := []struct { + name string + statusCode int + expected bool + }{ + { + name: "should return a retryable error", + statusCode: http.StatusInternalServerError, + expected: true, + }, + { + name: "should not return a retryable error", + statusCode: http.StatusNotFound, + expected: false, + }, + { + name: "should not return a retryable error when handling malformed data", + statusCode: http.StatusOK, + expected: false, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + etc := newESTestContext(t) + etc.ec.HTTP = fakeHTTPClient{respStatus: test.statusCode} + loader := MakeESLoader(etc.ec, "fakeIndexPattern", etc.location) + + _, err := loader(stdfields.StdMonitorFields{}) + + var loaderError LoaderError + require.ErrorAs(t, err, &loaderError) + require.Equal(t, loaderError.Retry, test.expected) + }) + } +} + +type fakeHTTPClient struct { + respStatus int +} + +func (fc fakeHTTPClient) Do(req *http.Request) (resp *http.Response, err error) { + return &http.Response{ + StatusCode: fc.respStatus, + Body: io.NopCloser(strings.NewReader("test response")), + }, nil +} + +func (fc fakeHTTPClient) CloseIdleConnections() { + // noop +} + type esTestContext struct { namespace string + ec *eslegclient.Connection esc *elasticsearch.Client loader StateLoader tracker *Tracker @@ -106,10 +163,12 @@ func newESTestContext(t *testing.T) *esTestContext { } namespace, _ := uuid.NewV4() esc := IntegApiClient(t) + ec := IntegES(t) etc := &esTestContext{ namespace: namespace.String(), esc: esc, - loader: IntegESLoader(t, fmt.Sprintf("synthetics-*-%s", namespace.String()), location), + ec: ec, + loader: IntegESLoader(t, ec, fmt.Sprintf("synthetics-*-%s", namespace.String()), location), location: location, } diff --git a/heartbeat/monitors/wrappers/monitorstate/testutil.go b/heartbeat/monitors/wrappers/monitorstate/testutil.go index 540983097587..28a6c2606557 100644 --- a/heartbeat/monitors/wrappers/monitorstate/testutil.go +++ b/heartbeat/monitors/wrappers/monitorstate/testutil.go @@ -33,8 +33,8 @@ import ( // Helpers for tests here and elsewhere -func IntegESLoader(t *testing.T, indexPattern string, location *config.LocationWithID) StateLoader { - return MakeESLoader(IntegES(t), indexPattern, location) +func IntegESLoader(t *testing.T, esc *eslegclient.Connection, indexPattern string, location *config.LocationWithID) StateLoader { + return MakeESLoader(esc, indexPattern, location) } func IntegES(t *testing.T) (esc *eslegclient.Connection) { diff --git a/heartbeat/monitors/wrappers/monitorstate/tracker.go b/heartbeat/monitors/wrappers/monitorstate/tracker.go index e350294e46e8..40a4e8f2ded1 100644 --- a/heartbeat/monitors/wrappers/monitorstate/tracker.go +++ b/heartbeat/monitors/wrappers/monitorstate/tracker.go @@ -62,7 +62,7 @@ func (t *Tracker) RecordStatus(sf stdfields.StdMonitorFields, newStatus StateSta t.mtx.Lock() defer t.mtx.Unlock() - state := t.GetCurrentState(sf) + state := t.GetCurrentState(sf, RetryConfig{}) if state == nil { state = newMonitorState(sf, newStatus, 0, t.flappingEnabled) logp.L().Infof("initializing new state for monitor %s: %s", sf.ID, state.String()) @@ -75,22 +75,33 @@ func (t *Tracker) RecordStatus(sf stdfields.StdMonitorFields, newStatus StateSta } func (t *Tracker) GetCurrentStatus(sf stdfields.StdMonitorFields) StateStatus { - s := t.GetCurrentState(sf) + s := t.GetCurrentState(sf, RetryConfig{}) if s == nil { return StatusEmpty } return s.Status } -func (t *Tracker) GetCurrentState(sf stdfields.StdMonitorFields) (state *State) { +type RetryConfig struct { + attempts int + waitFn func() time.Duration +} + +func (t *Tracker) GetCurrentState(sf stdfields.StdMonitorFields, rc RetryConfig) (state *State) { if state, ok := t.states[sf.ID]; ok { return state } - tries := 3 + // Default number of attempts + attempts := 3 + if rc.attempts != 0 { + attempts = rc.attempts + } + var loadedState *State var err error - for i := 0; i < tries; i++ { + var i int + for i = 0; i < attempts; i++ { loadedState, err = t.stateLoader(sf) if err == nil { if loadedState != nil { @@ -98,13 +109,22 @@ func (t *Tracker) GetCurrentState(sf stdfields.StdMonitorFields) (state *State) } break } + var loaderError LoaderError + if errors.As(err, &loaderError) && !loaderError.Retry { + logp.L().Warnf("could not load last externally recorded state: %v", loaderError) + break + } + // Default sleep time sleepFor := (time.Duration(i*i) * time.Second) + (time.Duration(rand.Intn(500)) * time.Millisecond) - logp.L().Warnf("could not load last externally recorded state, will retry again in %d milliseconds: %w", sleepFor.Milliseconds(), err) + if rc.waitFn != nil { + sleepFor = rc.waitFn() + } + logp.L().Warnf("could not load last externally recorded state, will retry again in %d milliseconds: %v", sleepFor.Milliseconds(), err) time.Sleep(sleepFor) } if err != nil { - logp.L().Warnf("could not load prior state from elasticsearch after %d attempts, will create new state for monitor: %s", tries, sf.ID) + logp.L().Warnf("could not load prior state from elasticsearch after %d attempts, will create new state for monitor: %s", i+1, sf.ID) } if loadedState != nil { diff --git a/heartbeat/monitors/wrappers/monitorstate/tracker_test.go b/heartbeat/monitors/wrappers/monitorstate/tracker_test.go index ec1217b86150..fd34371ce810 100644 --- a/heartbeat/monitors/wrappers/monitorstate/tracker_test.go +++ b/heartbeat/monitors/wrappers/monitorstate/tracker_test.go @@ -18,6 +18,7 @@ package monitorstate import ( + "errors" "testing" "time" @@ -131,3 +132,51 @@ func TestDeferredStateLoader(t *testing.T) { resState, _ = dsl(stdfields.StdMonitorFields{}) require.Equal(t, stateA, resState) } + +func TestStateLoaderRetry(t *testing.T) { + // While testing the sleep time between retries should be negligible + waitFn := func() time.Duration { + return time.Microsecond + } + + tests := []struct { + name string + retryable bool + rc RetryConfig + expectedCalls int + }{ + { + "should retry 3 times when fails with retryable error", + true, + RetryConfig{waitFn: waitFn}, + 3, + }, + { + "should not retry when fails with non-retryable error", + false, + RetryConfig{waitFn: waitFn}, + 1, + }, + { + "should honour the configured number of attempts when fails with retryable error", + true, + RetryConfig{attempts: 5, waitFn: waitFn}, + 5, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + calls := 0 + errorStateLoader := func(_ stdfields.StdMonitorFields) (*State, error) { + calls += 1 + return nil, LoaderError{err: errors.New("test error"), Retry: tt.retryable} + } + + mst := NewTracker(errorStateLoader, true) + mst.GetCurrentState(stdfields.StdMonitorFields{}, tt.rc) + + require.Equal(t, calls, tt.expectedCalls) + }) + } +} From 5f1e656e06fc07a0dee6a180bcb381853bd741dd Mon Sep 17 00:00:00 2001 From: Maurizio Branca Date: Mon, 4 Mar 2024 13:45:20 +0100 Subject: [PATCH 17/26] [AWS] [S3] Remove url.QueryUnescape() from aws-s3 input in polling mode (#38125) * Remove url.QueryUnescape() We introduced [^1] the `url.QueryUnescape()` function to unescape object keys from S3 notification in SQS messages. However, the object keys in the S3 list object responses do not require [^2] unescape. We must remove the unescape to avoid unintended changes to the S3 object key. [^1]: https://github.com/elastic/beats/pull/18370 [^2]: https://github.com/elastic/beats/issues/38012#issuecomment-1946440453 --------- Co-authored-by: Andrea Spacca --- CHANGELOG.next.asciidoc | 1 + x-pack/filebeat/input/awss3/s3.go | 12 ++---------- x-pack/filebeat/input/awss3/s3_test.go | 9 +++++++++ 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index eef3e2304167..7731d291ba49 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -97,6 +97,7 @@ fields added to events containing the Beats version. {pull}37553[37553] - [threatintel] MISP pagination fixes {pull}37898[37898] - Fix file handle leak when handling errors in filestream {pull}37973[37973] - Prevent HTTPJSON holding response bodies between executions. {issue}35219[35219] {pull}38116[38116] +- Fix "failed processing S3 event for object key" error on aws-s3 input when key contains the "+" character {issue}38012[38012] {pull}38125[38125] *Heartbeat* diff --git a/x-pack/filebeat/input/awss3/s3.go b/x-pack/filebeat/input/awss3/s3.go index 96be746c160a..5aa8d31e95de 100644 --- a/x-pack/filebeat/input/awss3/s3.go +++ b/x-pack/filebeat/input/awss3/s3.go @@ -8,7 +8,6 @@ import ( "context" "errors" "fmt" - "net/url" "sync" "time" @@ -208,14 +207,7 @@ func (p *s3Poller) GetS3Objects(ctx context.Context, s3ObjectPayloadChan chan<- // Metrics p.metrics.s3ObjectsListedTotal.Add(uint64(totListedObjects)) for _, object := range page.Contents { - // Unescape s3 key name. For example, convert "%3D" back to "=". - filename, err := url.QueryUnescape(*object.Key) - if err != nil { - p.log.Errorw("Error when unescaping object key, skipping.", "error", err, "s3_object", *object.Key) - continue - } - - state := newState(bucketName, filename, *object.ETag, p.listPrefix, *object.LastModified) + state := newState(bucketName, *object.Key, *object.ETag, p.listPrefix, *object.LastModified) if p.states.MustSkip(state, p.store) { p.log.Debugw("skipping state.", "state", state) continue @@ -240,7 +232,7 @@ func (p *s3Poller) GetS3Objects(ctx context.Context, s3ObjectPayloadChan chan<- s3ObjectHandler: s3Processor, s3ObjectInfo: s3ObjectInfo{ name: bucketName, - key: filename, + key: *object.Key, etag: *object.ETag, lastModified: *object.LastModified, listingID: listingID.String(), diff --git a/x-pack/filebeat/input/awss3/s3_test.go b/x-pack/filebeat/input/awss3/s3_test.go index 6f075a2f8541..b94ba7cfb09b 100644 --- a/x-pack/filebeat/input/awss3/s3_test.go +++ b/x-pack/filebeat/input/awss3/s3_test.go @@ -93,6 +93,11 @@ func TestS3Poller(t *testing.T) { Key: aws.String("key5"), LastModified: aws.Time(time.Now()), }, + { + ETag: aws.String("etag6"), + Key: aws.String("2024-02-08T08:35:00+00:02.json.gz"), + LastModified: aws.Time(time.Now()), + }, }, }, nil }) @@ -124,6 +129,10 @@ func TestS3Poller(t *testing.T) { GetObject(gomock.Any(), gomock.Eq(bucket), gomock.Eq("key5")). Return(nil, errFakeConnectivityFailure) + mockAPI.EXPECT(). + GetObject(gomock.Any(), gomock.Eq(bucket), gomock.Eq("2024-02-08T08:35:00+00:02.json.gz")). + Return(nil, errFakeConnectivityFailure) + s3ObjProc := newS3ObjectProcessorFactory(logp.NewLogger(inputName), nil, mockAPI, nil, backupConfig{}, numberOfWorkers) receiver := newS3Poller(logp.NewLogger(inputName), nil, mockAPI, mockPublisher, s3ObjProc, newStates(inputCtx), store, bucket, "key", "region", "provider", numberOfWorkers, pollInterval) require.Error(t, context.DeadlineExceeded, receiver.Poll(ctx)) From 4f46fcb888b7f12706d005b7e1d2d1dafcbe6fcd Mon Sep 17 00:00:00 2001 From: sharbuz <87968844+sharbuz@users.noreply.github.com> Date: Mon, 4 Mar 2024 17:05:07 +0200 Subject: [PATCH 18/26] increase the maximum timeout for x-pack-metricbeat (#38152) --- catalog-info.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalog-info.yaml b/catalog-info.yaml index 955c316d5aae..5e0f94fd2df0 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -637,7 +637,7 @@ spec: spec: # branch_configuration: "7.17" #TODO: uncomment after tests pipeline_file: ".buildkite/x-pack/pipeline.xpack.metricbeat.yml" - maximum_timeout_in_minutes: 240 + maximum_timeout_in_minutes: 480 provider_settings: trigger_mode: none # don't trigger jobs from github activity build_pull_request_forks: false From d23b4d33622137f7245b628d77d15f8ecfef3a0d Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Mon, 4 Mar 2024 12:51:37 -0500 Subject: [PATCH 19/26] Memory queue: cancel in-progress writes on queue closed, not producer closed (#38094) Fixes a race condition that could lead to incorrect event totals and occasional panics #37702. Once a producer sends a get request to the memory queue, it must wait on the response unless the queue itself is closed, otherwise it can return a false failure. The previous code mistakenly waited on the done signal for the current producer rather than the queue. This PR adds the queue's done signal to the producer struct, and waits on that once the insert request is sent. --- CHANGELOG.next.asciidoc | 1 + libbeat/publisher/queue/memqueue/produce.go | 39 ++++-- .../publisher/queue/memqueue/queue_test.go | 121 +++++++++++++++--- 3 files changed, 131 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 7731d291ba49..77e78868b6a8 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -96,6 +96,7 @@ fields added to events containing the Beats version. {pull}37553[37553] - Update github.com/lestrrat-go/jwx dependency. {pull}37799[37799] - [threatintel] MISP pagination fixes {pull}37898[37898] - Fix file handle leak when handling errors in filestream {pull}37973[37973] +- Fix a race condition that could crash Filebeat with a "negative WaitGroup counter" error {pull}38094[38094] - Prevent HTTPJSON holding response bodies between executions. {issue}35219[35219] {pull}38116[38116] - Fix "failed processing S3 event for object key" error on aws-s3 input when key contains the "+" character {issue}38012[38012] {pull}38125[38125] diff --git a/libbeat/publisher/queue/memqueue/produce.go b/libbeat/publisher/queue/memqueue/produce.go index 954ea055f4a4..5ddea468e4c6 100644 --- a/libbeat/publisher/queue/memqueue/produce.go +++ b/libbeat/publisher/queue/memqueue/produce.go @@ -36,9 +36,10 @@ type ackProducer struct { } type openState struct { - log *logp.Logger - done chan struct{} - events chan pushRequest + log *logp.Logger + done chan struct{} + queueDone <-chan struct{} + events chan pushRequest } // producerID stores the order of events within a single producer, so multiple @@ -58,9 +59,10 @@ type ackHandler func(count int) func newProducer(b *broker, cb ackHandler, dropCB func(interface{}), dropOnCancel bool) queue.Producer { openState := openState{ - log: b.logger, - done: make(chan struct{}), - events: b.pushChan, + log: b.logger, + done: make(chan struct{}), + queueDone: b.ctx.Done(), + events: b.pushChan, } if cb != nil { @@ -143,27 +145,40 @@ func (st *openState) Close() { func (st *openState) publish(req pushRequest) (queue.EntryID, bool) { select { case st.events <- req: - // If the output is blocked and the queue is full, `req` is written - // to `st.events`, however the queue never writes back to `req.resp`, - // which effectively blocks for ever. So we also need to select on the - // done channel to ensure we don't miss the shutdown signal. + // The events channel is buffered, which means we may successfully + // write to it even if the queue is shutting down. To avoid blocking + // forever during shutdown, we also have to wait on the queue's + // shutdown channel. select { case resp := <-req.resp: return resp, true - case <-st.done: + case <-st.queueDone: st.events = nil return 0, false } case <-st.done: st.events = nil return 0, false + case <-st.queueDone: + st.events = nil + return 0, false } } func (st *openState) tryPublish(req pushRequest) (queue.EntryID, bool) { select { case st.events <- req: - return <-req.resp, true + // The events channel is buffered, which means we may successfully + // write to it even if the queue is shutting down. To avoid blocking + // forever during shutdown, we also have to wait on the queue's + // shutdown channel. + select { + case resp := <-req.resp: + return resp, true + case <-st.queueDone: + st.events = nil + return 0, false + } case <-st.done: st.events = nil return 0, false diff --git a/libbeat/publisher/queue/memqueue/queue_test.go b/libbeat/publisher/queue/memqueue/queue_test.go index 141514483f33..53f8da4b77c6 100644 --- a/libbeat/publisher/queue/memqueue/queue_test.go +++ b/libbeat/publisher/queue/memqueue/queue_test.go @@ -27,8 +27,7 @@ import ( "testing" "time" - "gotest.tools/assert" - + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/elastic/beats/v7/libbeat/publisher/queue" @@ -77,17 +76,17 @@ func TestProduceConsumer(t *testing.T) { t.Run("flush", testWith(makeTestQueue(bufferSize, batchSize/2, 100*time.Millisecond))) } -// TestProducerDoesNotBlockWhenCancelled ensures the producer Publish -// does not block indefinitely. +// TestProducerDoesNotBlockWhenQueueClosed ensures the producer Publish +// does not block indefinitely during queue shutdown. // -// Once we get a producer `p` from the queue we want to ensure +// Once we get a producer `p` from the queue `q` we want to ensure // that if p.Publish is called and blocks it will unblock once -// p.Cancel is called. +// `q.Close` is called. // // For this test we start a queue with size 2 and try to add more -// than 2 events to it, p.Publish will block, once we call p.Cancel, +// than 2 events to it, p.Publish will block, once we call q.Close, // we ensure the 3rd event was not successfully published. -func TestProducerDoesNotBlockWhenCancelled(t *testing.T) { +func TestProducerDoesNotBlockWhenQueueClosed(t *testing.T) { q := NewQueue(nil, nil, Settings{ Events: 2, // Queue size @@ -138,8 +137,12 @@ func TestProducerDoesNotBlockWhenCancelled(t *testing.T) { time.Millisecond, "the first two events were not successfully published") - // Cancel the producer, this should unblock its Publish method - p.Cancel() + // Close the queue, this should unblock the pending Publish call. + // It's not enough to just cancel the producer: once the producer + // has successfully sent a request to the queue, it must wait for + // the response unless the queue shuts down, otherwise the pipeline + // event totals will be wrong. + q.Close() require.Eventually( t, @@ -149,6 +152,88 @@ func TestProducerDoesNotBlockWhenCancelled(t *testing.T) { "test not flagged as successful, p.Publish likely blocked indefinitely") } +func TestProducerClosePreservesEventCount(t *testing.T) { + // Check for https://github.com/elastic/beats/issues/37702, a problem + // where canceling a producer while it was waiting on a response + // to an insert request could lead to inaccurate event totals. + + var activeEvents atomic.Int64 + + q := NewQueue(nil, nil, + Settings{ + Events: 3, // Queue size + MaxGetRequest: 2, + FlushTimeout: 10 * time.Millisecond, + }, 1) + + p := q.Producer(queue.ProducerConfig{ + ACK: func(count int) { + activeEvents.Add(-int64(count)) + }, + OnDrop: func(e interface{}) { + //activeEvents.Add(-1) + }, + DropOnCancel: false, + }) + + // Asynchronously, send 4 events to the queue. + // Three will be enqueued, and one will be buffered, + // until we start reading from the queue. + // This needs to run in a goroutine because the buffered + // event will block until the queue handles it. + var wgProducer sync.WaitGroup + wgProducer.Add(1) + go func() { + for i := 0; i < 4; i++ { + event := i + // For proper navigation of the race conditions inherent to this + // test: increment active events before the publish attempt, then + // decrement afterwards if it failed (otherwise the event count + // could become negative even under correct queue operation). + activeEvents.Add(1) + _, ok := p.Publish(event) + if !ok { + activeEvents.Add(-1) + } + } + wgProducer.Done() + }() + + // This sleep is regrettable, but there's no deterministic way to know when + // the producer code has buffered an event in the queue's channel. + // However, the test is written to produce false negatives only: + // - If this test fails, it _always_ indicates a bug. + // - If there is a bug, this test will _often_ fail. + time.Sleep(20 * time.Millisecond) + + // Cancel the producer, then read and acknowledge two batches. If the + // Publish calls and the queue code are working, activeEvents should + // _usually_ end up as 0, but _always_ end up non-negative. + p.Cancel() + + // The queue reads also need to be done in a goroutine, in case the + // producer cancellation signal went through before the Publish + // requests -- if only 2 events entered the queue, then the second + // Get call will block until the queue itself is cancelled. + go func() { + for i := 0; i < 2; i++ { + batch, err := q.Get(2) + // Only error to worry about is queue closing, which isn't + // a test failure. + if err == nil { + batch.Done() + } + } + }() + + // One last sleep to let things percolate, then we close the queue + // to unblock any helpers and verify that the final active event + // count isn't negative. + time.Sleep(10 * time.Millisecond) + q.Close() + assert.False(t, activeEvents.Load() < 0, "active event count should never be negative") +} + func TestQueueMetricsDirect(t *testing.T) { eventsToTest := 5 maxEvents := 10 @@ -190,7 +275,7 @@ func queueTestWithSettings(t *testing.T, settings Settings, eventsToTest int, te // Read events, don't yet ack them batch, err := testQueue.Get(eventsToTest) - assert.NilError(t, err, "error in Get") + assert.NoError(t, err, "error in Get") t.Logf("Got batch of %d events", batch.Count()) queueMetricsAreValid(t, testQueue, 5, settings.Events, 5, fmt.Sprintf("%s - Producer Getting events, no ACK", testName)) @@ -206,7 +291,7 @@ func queueMetricsAreValid(t *testing.T, q queue.Queue, evtCount, evtLimit, occup // wait briefly to avoid races across all the queue channels time.Sleep(time.Millisecond * 100) testMetrics, err := q.Metrics() - assert.NilError(t, err, "error calling metrics for test %s", test) + assert.NoError(t, err, "error calling metrics for test %s", test) assert.Equal(t, testMetrics.EventCount.ValueOr(0), uint64(evtCount), "incorrect EventCount for %s", test) assert.Equal(t, testMetrics.EventLimit.ValueOr(0), uint64(evtLimit), "incorrect EventLimit for %s", test) assert.Equal(t, testMetrics.UnackedConsumedEvents.ValueOr(0), uint64(occupied), "incorrect OccupiedRead for %s", test) @@ -266,18 +351,18 @@ func TestEntryIDs(t *testing.T) { for i := 0; i < entryCount; i++ { batch, err := q.Get(1) - assert.NilError(t, err, "Queue read should succeed") + assert.NoError(t, err, "Queue read should succeed") assert.Equal(t, batch.Count(), 1, "Returned batch should have 1 entry") metrics, err := q.Metrics() - assert.NilError(t, err, "Queue metrics call should succeed") + assert.NoError(t, err, "Queue metrics call should succeed") assert.Equal(t, metrics.OldestEntryID, queue.EntryID(i), fmt.Sprintf("Oldest entry ID before ACKing event %v should be %v", i, i)) batch.Done() waiter.waitForEvents(1) metrics, err = q.Metrics() - assert.NilError(t, err, "Queue metrics call should succeed") + assert.NoError(t, err, "Queue metrics call should succeed") assert.Equal(t, metrics.OldestEntryID, queue.EntryID(i+1), fmt.Sprintf("Oldest entry ID after ACKing event %v should be %v", i, i+1)) @@ -297,7 +382,7 @@ func TestEntryIDs(t *testing.T) { for i := 0; i < entryCount; i++ { batch, err := q.Get(1) - assert.NilError(t, err, "Queue read should succeed") + assert.NoError(t, err, "Queue read should succeed") assert.Equal(t, batch.Count(), 1, "Returned batch should have 1 entry") batches = append(batches, batch) } @@ -318,7 +403,7 @@ func TestEntryIDs(t *testing.T) { // the slight nondeterminism. time.Sleep(1 * time.Millisecond) metrics, err := q.Metrics() - assert.NilError(t, err, "Queue metrics call should succeed") + assert.NoError(t, err, "Queue metrics call should succeed") assert.Equal(t, metrics.OldestEntryID, queue.EntryID(0), fmt.Sprintf("Oldest entry ID after ACKing event %v should be 0", i)) } @@ -326,7 +411,7 @@ func TestEntryIDs(t *testing.T) { batches[0].Done() waiter.waitForEvents(100) metrics, err := q.Metrics() - assert.NilError(t, err, "Queue metrics call should succeed") + assert.NoError(t, err, "Queue metrics call should succeed") assert.Equal(t, metrics.OldestEntryID, queue.EntryID(100), fmt.Sprintf("Oldest entry ID after ACKing event 0 should be %v", queue.EntryID(entryCount))) From 0797a64da081ae1609fdbf831b6b3878b4075e88 Mon Sep 17 00:00:00 2001 From: Dan Kortschak <90160302+efd6@users.noreply.github.com> Date: Wed, 6 Mar 2024 12:12:04 +1030 Subject: [PATCH 20/26] x-pack/filebeat/input/{cel,websocket}: prevent addition of regexp extension when no patterns are provided (#38181) In the case of cel, the extension was always being added, and then was conditionally added when a pattern was configured, resulting in a failed configuration. In the case of websocket the conditional addition was not being performed, so no failure occurred, but additional look-up load was being added. So unify the approaches to the correct conditional addition. --- CHANGELOG.next.asciidoc | 1 + x-pack/filebeat/input/cel/config_test.go | 14 ++++++++++++++ x-pack/filebeat/input/cel/input.go | 1 - x-pack/filebeat/input/websocket/cel.go | 4 +++- x-pack/filebeat/input/websocket/config_test.go | 13 +++++++++++++ 5 files changed, 31 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 77e78868b6a8..f11f88e491c9 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -99,6 +99,7 @@ fields added to events containing the Beats version. {pull}37553[37553] - Fix a race condition that could crash Filebeat with a "negative WaitGroup counter" error {pull}38094[38094] - Prevent HTTPJSON holding response bodies between executions. {issue}35219[35219] {pull}38116[38116] - Fix "failed processing S3 event for object key" error on aws-s3 input when key contains the "+" character {issue}38012[38012] {pull}38125[38125] +- Fix duplicated addition of regexp extension in CEL input. {pull}38181[38181] *Heartbeat* diff --git a/x-pack/filebeat/input/cel/config_test.go b/x-pack/filebeat/input/cel/config_test.go index e4c98b78dc5e..7acf74df08ca 100644 --- a/x-pack/filebeat/input/cel/config_test.go +++ b/x-pack/filebeat/input/cel/config_test.go @@ -12,6 +12,7 @@ import ( "os" "reflect" "testing" + "time" "github.com/google/go-cmp/cmp" "golang.org/x/oauth2/google" @@ -38,6 +39,19 @@ func TestGetProviderIsCanonical(t *testing.T) { } } +func TestRegexpConfig(t *testing.T) { + cfg := config{ + Interval: time.Minute, + Program: `{}`, + Resource: &ResourceConfig{URL: &urlConfig{URL: &url.URL{}}}, + Regexps: map[string]string{"regex_cve": `[Cc][Vv][Ee]-[0-9]{4}-[0-9]{4,7}`}, + } + err := cfg.Validate() + if err != nil { + t.Errorf("failed to validate config with regexps: %v", err) + } +} + func TestIsEnabled(t *testing.T) { type enabler interface { isEnabled() bool diff --git a/x-pack/filebeat/input/cel/input.go b/x-pack/filebeat/input/cel/input.go index 12dd4c4dcecf..edd7c0530183 100644 --- a/x-pack/filebeat/input/cel/input.go +++ b/x-pack/filebeat/input/cel/input.go @@ -918,7 +918,6 @@ func newProgram(ctx context.Context, src, root string, client *http.Client, limi lib.Debug(debug(log, trace)), lib.File(mimetypes), lib.MIME(mimetypes), - lib.Regexp(patterns), lib.Limit(limitPolicies), lib.Globals(map[string]interface{}{ "useragent": userAgent, diff --git a/x-pack/filebeat/input/websocket/cel.go b/x-pack/filebeat/input/websocket/cel.go index 11c2e7ad8f13..0938da053535 100644 --- a/x-pack/filebeat/input/websocket/cel.go +++ b/x-pack/filebeat/input/websocket/cel.go @@ -63,11 +63,13 @@ func newProgram(ctx context.Context, src, root string, patterns map[string]*rege lib.Try(), lib.Debug(debug(log)), lib.MIME(mimetypes), - lib.Regexp(patterns), lib.Globals(map[string]interface{}{ "useragent": userAgent, }), } + if len(patterns) != 0 { + opts = append(opts, lib.Regexp(patterns)) + } env, err := cel.NewEnv(opts...) if err != nil { diff --git a/x-pack/filebeat/input/websocket/config_test.go b/x-pack/filebeat/input/websocket/config_test.go index 021bf89056f0..c1aaac973283 100644 --- a/x-pack/filebeat/input/websocket/config_test.go +++ b/x-pack/filebeat/input/websocket/config_test.go @@ -6,6 +6,7 @@ package websocket import ( "fmt" + "net/url" "testing" "github.com/stretchr/testify/assert" @@ -119,3 +120,15 @@ func TestConfig(t *testing.T) { }) } } + +func TestRegexpConfig(t *testing.T) { + cfg := config{ + Program: `{}`, + URL: &urlConfig{URL: &url.URL{Scheme: "ws"}}, + Regexps: map[string]string{"regex_cve": `[Cc][Vv][Ee]-[0-9]{4}-[0-9]{4,7}`}, + } + err := cfg.Validate() + if err != nil { + t.Errorf("failed to validate config with regexps: %v", err) + } +} From a9dfc67bb4ab36baefe2838b044f8b64a107881e Mon Sep 17 00:00:00 2001 From: Kush Rana <89848966+kush-elastic@users.noreply.github.com> Date: Wed, 6 Mar 2024 10:55:45 +0530 Subject: [PATCH 21/26] [Metricbeat][Postgresql][Database] Fix fields not being parsed correctly (#37720) * change blk_read_time and blk_write_time from long to double type * add changelog entry * remove unnecessary changes --- CHANGELOG.next.asciidoc | 1 + metricbeat/docs/fields.asciidoc | 4 ++-- metricbeat/module/postgresql/database/_meta/fields.yml | 4 ++-- metricbeat/module/postgresql/database/data.go | 4 ++-- metricbeat/module/postgresql/fields.go | 2 +- 5 files changed, 8 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index f11f88e491c9..2c2a762be6ca 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -109,6 +109,7 @@ fields added to events containing the Beats version. {pull}37553[37553] *Metricbeat* +- Fix fields not being parsed correctly in postgresql/database {issue}25301[25301] {pull}37720[37720] *Osquerybeat* diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index d96172d0bfca..435760b7406e 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -56448,7 +56448,7 @@ type: long Time spent reading data file blocks by backends in this database, in milliseconds. -type: long +type: double -- @@ -56458,7 +56458,7 @@ type: long Time spent writing data file blocks by backends in this database, in milliseconds. -type: long +type: double -- diff --git a/metricbeat/module/postgresql/database/_meta/fields.yml b/metricbeat/module/postgresql/database/_meta/fields.yml index 2b08d1630def..7eb5ceca4f43 100644 --- a/metricbeat/module/postgresql/database/_meta/fields.yml +++ b/metricbeat/module/postgresql/database/_meta/fields.yml @@ -36,12 +36,12 @@ that a read was not necessary (this only includes hits in the PostgreSQL buffer cache, not the operating system's file system cache). - name: blocks.time.read.ms - type: long + type: double description: > Time spent reading data file blocks by backends in this database, in milliseconds. - name: blocks.time.write.ms - type: long + type: double description: > Time spent writing data file blocks by backends in this database, in milliseconds. diff --git a/metricbeat/module/postgresql/database/data.go b/metricbeat/module/postgresql/database/data.go index 99d2bdb643fb..5a662a8aedbb 100644 --- a/metricbeat/module/postgresql/database/data.go +++ b/metricbeat/module/postgresql/database/data.go @@ -37,8 +37,8 @@ var schema = s.Schema{ "read": c.Int("blks_read"), "hit": c.Int("blks_hit"), "time": s.Object{ - "read": s.Object{"ms": c.Int("blk_read_time")}, - "write": s.Object{"ms": c.Int("blk_write_time")}, + "read": s.Object{"ms": c.Float("blk_read_time")}, + "write": s.Object{"ms": c.Float("blk_write_time")}, }, }, "rows": s.Object{ diff --git a/metricbeat/module/postgresql/fields.go b/metricbeat/module/postgresql/fields.go index cd679d7edc24..2962ab237f21 100644 --- a/metricbeat/module/postgresql/fields.go +++ b/metricbeat/module/postgresql/fields.go @@ -32,5 +32,5 @@ func init() { // AssetPostgresql returns asset data. // This is the base64 encoded zlib format compressed contents of module/postgresql. func AssetPostgresql() string { - return "eJzUWk+P47oNv8+nIN7l7RbZoL3OoUCxr0AX6O7bh92ix4CR6ViILHklOZn00xeU5D+xncwkY8+2Oc0kFvkTKZE/kv4Aezo9QmWc31lyP9QDgJde0SP88jV++e2Pf/7yAJCRE1ZWXhr9CH99AAD4TN5K4UAYpUh4yiC3poRuHTiyB7Ju/QDgCmP9Rhidy90j5KgcPQBYUoSOHmGHDwC5JJW5xyD8A2gsaQCNP/5U8fPW1FX6ZgIaf3o4yoh0nX7r6+nrQuHlQfpT+8OUtisa+fO7JsiMqEvSHiqyyQZQWSPIuRUb4ij1DqTOjS2RZbAZkO3nDfiCQNTWkvZnchtsYHLwBfqewFoUgA6cR0+AOmvWw4+a7GkNH1v/bE9nMsPvjKXabXj1plGy7j127qLmMzRh34wZetyio7WR2dkDjTmV0bvBD1csGqz66be4cWqlgy+kgy2KPekMJB9DreM2vVlfB8b/TiLb0+lo7BD1M+C+YEkzoKtms9bXeDSgMVqHZFpz7ciul/AVCwZldjvKQOpwul+EZcI/N/jgDq1YVUqKcBk3r1PekxTv6cD3LwAjlCTt15hllpy7Dcqnr5DWNYCitDsxFMb52+3xD+N8kNNiaJVHuSuOV5YqY2NUAgRLnCkIfvvyDZQx+7rixfHxDW/pKk6WNNPx/f7xK7A40HW5JRud2DOkdFA7Dpq5sSBMWda68fdR+iLYdiQ02XoFxsKHv4DMAeFfWj6BM2JPSShd8EVavOEd3biXU9X5ICWFxu2cH53cKgqWcoCWAGtvDijqugSFtRYF2VX/y6Oxe7KrkR5ldlKgYpe2h78TMPVrkgQVWlSKVPsFw+N0q4fhCOBopedHkiNam4qCxL4yUodfnUfr62oFR1SWBMkDf3tkwqEzsiFBHlFFYeuRkr8/edJOGu2gxBNY2knnySZ8LvoYs0zyNlC1IT4Y8br/ArLJU5qhv9WzsiQ4FqTjXU5kAI6RB/C1WoFc03rVPDQZCEZi+blIWKa34i1qxyzB6DfYzq/toe3p7e9xGmSgNYvBa2+SOkVSRlHhue2N5UseKZl0oM0QSmJ01HOQQufHsqb3GCRvRIF6N81kXrvJCJ1xBFhR0wUwR5RejuJsxLE1RhHqG6HYmth+IxLVWj6pBKMBQRmxv2Km23R/TEfOHIhDUzLEkEd10fOAqo7hs2PDI6EAf0r+foTvBfX3RE8k6rAXTIR9crXM1HhtYwVORQiajt0lL0sc5va+KJC6f6lGkiXbtffACra1v3SS+dO55oYNDVDAO9wGSvCe8UjX3R8nS6nQMndJ6yZBnAGmJ0GVB6PbFBjEcWEW9sff9JUL5BQ8KRc1kLVmIl3wTnJ0vkJfQF7rRpRSVx3NSz6crZkWnUmHW0XZ0B4td+JLYlHsm9JNkuPfm3Vxn8+RveClG28oPfnhpfjVQcnMj7NuV31+6oXBFC/DolBBjuRydewGUbYzXCNSA99M4wuur1m4W4H03eIxQelCa+BzHNei2GsxbUOHYRn+rGH+jdJDWBcZsIxH71oQew7APcSviLSOXRTBcIg4FlIUQzgjEC1t2UWO9JpuyDePXjovhQPcmtq3yiPFS5SuzfeubXP0uxaRbg/d2vQsGpjjWHN756Jjkm7tREFZrUYR4d664kssJ0wOreS+vnguCzwQbIk0VGRzY8tLx7OP1NKPmpxfAGkreSakXpbk1sFf63JYJUW4uTJ445X7bjwqwNLUOsYkJjIJpIsYXRVKtxj0OXRylOb6sgU3kprOJB+9Y0GWIJcqlUm8Ac88yXCg3a9YcCmVko6E0dmFMmBsCHfS4v/ZDoy/sEbL/8S2wg3G2NZ5Ttate0aZ/fQmHa27stryLvp+uI5tgrjOj2p7mg6KL8C2yWulZgcYzuaFQO28qSrKACEAYHM6gRq2FNgTyPH5KTDrXRgDJepTu42re0xJavlzIS0JzsehEXWVKQ2gbXK+Agt5oIUSTOhNQ1hAegfmqCEoD1wT3mljS1RqyOLggh8L1JkKXjaOAkPoKr9Ga2aYS0Zd072Y91dthEoZgUukpcaBrYbLxZ/bWHI0ax+A6WNDpFwsKhLFOXJ4DFQzKB1TqmY68BpK9bsmsOYYZkuNvG6q1Hzz4SizPrbzKVA7+ZlkVBMob6dSP2/2E5ovfw6E1xVoKWNvmNqKS/25nzwNWgGVlT/dAjjchI3JN0nkArkzCe5VK+MJQjtPe65X6NbClOUoOcwQKXs62lq3Z/UzohoxXAwXZ3itUYpt8HMRMwq+sHipv7VVRuy5AsD5oyyzuaQAWMEI7VVIxRLeDnmxjytE3DxkNlQM8tSQ25gjQKAoODxOtbrRh7kT8xMMjS3QxFQX7QnehZ0arVigUHVGDgrZNY66lwtGgs81s1heYCqyGJoY7uQ8lb+6QKTTf/Hp91ctyrsPnr5UMtwx6+J8FmsCFtwkkIgsmXh76oLB8ASsphpxLyD/vQ1dLQZftSOW/FY7subIt9DXVi9RhpsjX8EovUnfodP3ohsZwOXkRbEUtiT8TmhSO7KLtC8YWyP9TnB1lS3CYQO2JPxOaBkpWgxaEn47NGF0rqRYoJ5vcAjUgjgvZjUxF2k1xuGsJWEOZE8N3pHEZ2gLlZWxaE/r0OiYP4k18lMjRVh69gzA30Z1PowEoSUQptZh/mhph5ZLvPAeyLGITYbzJZz2RlIbOO9ovVtz4rRxzsY1oyuk3r1fhRH6uYIwvjS7DSvYTNkNwJG/3OzujL49+dmMPmyIhUzQa94NLTh2weWzwy65wwXPeDEwkeSCJeycEWYhCc7PFhvJkJGPZcLL4sX/TpU+KtfbeeFr6/WzF0LjGIr/Ci+qTdfxE2+HnkmV+mDSqzTNC6FBbvc6qKhqqB3u4iuhPlyFwLhueB+0G5i+bq7yli8Ydg2sYBWLkbBPTKLf/H3V/psittYwnPqezWPns1eHpfXnNb2enm4cPPwRB6vDdediBSq1QDZtm9XRtO2kw9ZXjcscZ9Yso6+z9QtHL1mcr2W4obNPgzpY/SKmqflqrbux+nMAS6lnhPdZalnW5awA8WlOgPg0O0DCiya8/dx9JtRzonM+y+gwH76vpqpVTFHOo87QZpDRQXZZq9d96CN94QgxQi+pNPa0jo3TGbtOw+uTOrOhhRC7NbEflMZ3z5r4HOeMDbsXAA09rvuAZtJ6OVuN+QKsSeGdcBO5fzu4gynuS+EqI1AteFqD/Fcf1ohywbM6hnnPUY0wlz2pY6R3HtQIdtlzOgZ75zHlMnNJ/7P8V7s/gFzWoCOc0/b8bwAAAP//1pQyXQ==" + return "eJzUWk+P47oNv8+nIN7l7RbZoL3OoUCxr0AX6O7bh92ix4CR6ViILHklOZn00xeU5D+xncwkY8+2Oc0kFvkTSZE/Uv4Aezo9QmWc31lyP9QDgJde0SP88jV++e2Pf/7yAJCRE1ZWXhr9CH99AAD4TN5K4UAYpUh4yiC3poRuHTiyB7Ju/QDgCmP9Rhidy90j5KgcPQBYUoSOHmGHDwC5JJW5xyD8A2gsaQCNP/5U8fPW1FX6ZgIaf3o4yoh0nX7r6+nrQuHlQfpT+8OUtisa+fO7JsiMqEvSHiqyyQZQWSPIuRUb4ij1DqTOjS2RZbAZkO3nDfiCQNTWkvZnchtsYHLwBfqewFoUgA6cR0+AOmvWw4+a7GkNH1v/bE9nMsPvjKXabXj1plGy7j127qLmMzRh34wZetyio7WR2dkDjTmV0bvBD1csGqz66be4cWqlgy+kgy2KPekMJIeh1nGb3qyvA+N/J5Ht6XQ0doj6GXBfsKQZ0FWzWetrDA1ojNYhmdZcO7LrJXzFgkGZ3Y4ykDpE94uwTPjnBh/coRWrSkkRDuPmdcp7kuI5Hfj+BWCEkqT9GrPMknO3Qfn0FdK6BlCUdieGwjh/uz3+YZwPcloMrfIod8X5ylJlbMxKgGCJKwXBb1++gTJmX1e8OD6+4S1dxcmSZgrf7x+/AosDXZdbstGJPUNKB7XjpJkbC8KUZa0bfx+lL4JtR0KTrVdgLHz4C8gcEP6l5RM4I/aUhNIFX6TFG97RjXs5VZ0PUlFo3M710cmtomApB2gJsPbmgKKuS1BYa1GQXfW/PBq7J7sa6VFmJwUqdmkb/J2AqV+TJKjQolKk2i8YHpdbPUxHAEcrPT+SHNHaVBQk9pWROvzqPFpfVys4orIkSB742yMTDp2RDQXyiCoKW4+U/P3Jk3bSaAclnsDSTjpPNuFz0ceYZZK3gapN8cGI1/0XkE1GaYb+Vs/KkuBYkI5nOZEBOEYewMdqBXJN61Xz0GQiGInl5yJhmd6Kt6gdswSj32A7v7ZB29Pb3+M0yEBrFoPXniR1iqSMosJz2xvLhzxSMulAmyGUxOio5yCFzo9lTe8xSN6IAvVumsm8dpMROuMIsKKmC2COKL0c5dmIY2uMItQ3QrE1sf1GJKq1fFIJRgOCMmJ/xUy36f6YQs4ciFNTMsSQR3XZ84CqjumzY8MjoQB/Sv5+hO8F9fdETyTqsBdMhH1ytczUeG1jBS5FCJqO3SEvSxzW9r4okLp/qEaSJdu198AKtrW/FMn86Vxzw4YGKOAdbgMleM94pOvOj5OlVGiZu6R1kyDOANOToMqD0W0JDOK4MQv742/6ygVyCZ6UixrIWjNRLngnOTpfoS8gr3UjSqmrjuYlH87WTIvOpMOtomxoj5Y78SGxKPZN6ybJ8e/NurjP58he8NKNJ5Se/PBQ/OqgZObHVbfrPj/10mDKl2FR6CBHcrk7doMs2xmuEamBT6bxBffXLNytQPpu8ZigdKk18DnOa1HstZy2ocOwDX/WMP9G6SGsiwxYxtC7lsSeA3AP8SsirWMXRTCcIo6FFMUQzghES1t2kSO9ZhryzaOXzkvhALem9q3ySPESpWvrvWvHHP2pRaTbQ7c2M4sG5jjX3D656JikWztRUFarUUa4t6/4EtsJk0Mrua8vxmWBB4ItkYaKbG5seSk8+0gt/ajJ+QWQtpJnQuplSW4d/LUuh11ShJsrgzceue/GowIsTa1jTmIik0C6iNFVoXWLSZ9TJ2dp7i9bcCOpKSY59I4FWYJcqtQm8QY88yTDiXa/YsGlVEo6EkZnF9qAsSHcSYv/Zzsw/sIaLf8Txwo3GGNb5zlZt+4ZZfboTTpad2W15V30/XAd2wRxnR/V9jSdFF+AbZPXSs0OMMTmhUTtvKkqygAhAGBzOoEathTYE8hx/BSY9Q6MgRL1qd3G1T2mIrV8XEhLgutxGERdZUoDaJucj8BCHmihBBN60xAWkN6BOWoIygPXhHfa2BKVGrI4uODHAnWmgpeNo8AQus6v0ZoZ5pJR1/Qs5v1VG6FSRuASZalxYKvhcvPnNpYczToHYPrYECkXm4pEcY6cHgPVDErHlKq5HXgNpfpdE1hzDHdLjbzuVqn55sNRZn1s57dA7c3PJKOaQHk7lfp5dz9h+PLnQHhdgZYy9oaprbg0n/vJt0EroLLyp1sAh5OwMfkmiVygdibBvW5lfIPQ3qc9Nyt0a2HKclQcZsiUPR1tr9uz+hlRjRguposzvNYoxTb4uYgZBR9YvDTf2ioj9twB4PxZltlcUgCsYIT2KqRiCW+HutjHFTJuHiobKgZ5ashtrBEgUBScHqdG3ejDvRPzEwyDLdDEVBftCd6FnRqtWKBQdUYOCtkNjrqXC0aCzzWzWF5gKrIYhhju5DyVv7pApNN/8en3Vy3Kuw+evtQyZKbeqnsqWuwKWHRTQiK2ZOTtqUsHwxhYTY3iXkD/e1u62g6+ck8s+632ZM2RT6KvrV6iFTdHPoZRelPCw7TvRacygMvJi2IpbEn4ndCkdmQXGWEwtkb6neDqKluExwZsSfid0DJStBi0JPx2aMLoXEmxQE/f4BCoBXFtzGpiPtJqjBe0loQ5kD01eEcSn6EuVFbGoj2tw7Bj/kLWyE/DFGHp2RiAv416fRgJQksgTK3DHaSlHVpu88K7IMciDhrOl3DpG0lt4Lyj9W7NxdPGuzbuG10h9e79KlyjnysIV5hmt2EFmym7ATjylwfendG3Jz+b0YdDsVAJegO8oQXHLrgcO+ySO1zwjBcDG0kuWMLOGWEWiuD8jLGRDBn52Cq8LF/873Tqo5a9vTN8bc9+9lJovIriv8LLatO9/MQbomdSpT6Y9DpN81JokNu9EiqqGmqHu/haqA9HIXCuG94J7S5NX3e38pYvGXZDrGAVi5G0T9xGv/k7q/23RWytYXjze3YnO5+9OiytP6/p9fR04+XDH/FydbjuXKxApRaopu3AOpq2ve2w9VXjMseZtcro62z9Qugli/OxDCd09huhDla/iWm6vlrr7mr9OYCl1DPC+yy1LOtyVoD4NCdAfJodIOFFE94ed58J9ZzonM8yOsyH76upahVLlPOoM7QZZHSQXdXqzR/6SF94jRihl1Qae1rH4emMk6fh8UnT2TBCiBObOBNKV3jPmvgc54xDuxcADXOu+4Bm0no5W4/5AqxJ4Z1wE7l/O7iDm9yXwlVGoFowWoP8VwdrRLlgrI5h3hOqEeaykTpGemegRrDLxukY7J1hym3mkv5n+a92fwC5rEFHOKft+d8AAAD//5VSM/M=" } From 00e38f5a5f575c48746bad013c01106d57aa60a2 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 6 Mar 2024 09:36:00 +0100 Subject: [PATCH 22/26] github-action: install synthetics for running the UTs in x-pack/heartbeat (#38185) * install globally --- .github/workflows/macos-xpack-heartbeat.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/macos-xpack-heartbeat.yml b/.github/workflows/macos-xpack-heartbeat.yml index 502d10c1a3ea..8a0c6c1897d2 100644 --- a/.github/workflows/macos-xpack-heartbeat.yml +++ b/.github/workflows/macos-xpack-heartbeat.yml @@ -17,15 +17,17 @@ jobs: macos: runs-on: macos-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v3 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v4 with: go-version-file: .go-version - uses: actions/setup-python@v4 with: python-version: '3.9' - name: Install dependencies - run: go install github.com/magefile/mage + run: | + go install github.com/magefile/mage + cd ${{ env.BEAT_MODULE }} && npm install -g @elastic/synthetics - name: Run build run: cd ${{ env.BEAT_MODULE }} && mage build - name: Run test From ae312c5b288c8377ed1a87adda80e153c363cea5 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Wed, 6 Mar 2024 06:40:00 -0500 Subject: [PATCH 23/26] [Filebeat] ETW input - add basic metrics (#38017) Add input metrics to for ETW. Remove the very verbose debug message that was in the hot path for processing each event. I think more metrics taken from the ETW session itself should be added separately (e.g. events lost, buffers lost). This lays the ground works needed to expose those metrics. Closes #38007 --- .../filebeat/docs/inputs/input-etw.asciidoc | 100 ++++++++++++++---- x-pack/filebeat/input/etw/input.go | 65 +++++++++++- x-pack/filebeat/input/etw/input_test.go | 5 + 3 files changed, 148 insertions(+), 22 deletions(-) diff --git a/x-pack/filebeat/docs/inputs/input-etw.asciidoc b/x-pack/filebeat/docs/inputs/input-etw.asciidoc index 9ace3fdcc1b9..27dadaca2b74 100644 --- a/x-pack/filebeat/docs/inputs/input-etw.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-etw.asciidoc @@ -11,13 +11,29 @@ beta[] -https://learn.microsoft.com/en-us/windows/win32/etw/event-tracing-portal[Event Tracing for Windows] is a powerful logging and tracing mechanism built into the Windows operating system. It provides a detailed view of application and system behavior, performance issues, and runtime diagnostics. Trace events contain an event header and provider-defined data that describes the current state of an application or operation. You can use the events to debug an application and perform capacity and performance analysis. - -The ETW input can interact with ETW in three distinct ways: it can create a new session to capture events from user-mode providers, attach to an already existing session to collect ongoing event data, or read events from a pre-recorded .etl file. This functionality enables the module to adapt to different scenarios, such as real-time event monitoring or analyzing historical data. - -This input currently supports manifest-based, MOF (classic) and TraceLogging providers while WPP providers are not supported. https://learn.microsoft.com/en-us/windows/win32/etw/about-event-tracing#types-of-providers[Here] you can find more information about the available types of providers. - -It has been tested in every Windows versions supported by Filebeat, starting from Windows 8.1 and Windows Server 2016. In addition, administrative privileges are required in order to control event tracing sessions. +https://learn.microsoft.com/en-us/windows/win32/etw/event-tracing-portal[Event +Tracing for Windows] is a powerful logging and tracing mechanism built into the +Windows operating system. It provides a detailed view of application and system +behavior, performance issues, and runtime diagnostics. Trace events contain an +event header and provider-defined data that describes the current state of an +application or operation. You can use the events to debug an application and +perform capacity and performance analysis. + +The ETW input can interact with ETW in three distinct ways: it can create a new +session to capture events from user-mode providers, attach to an already +existing session to collect ongoing event data, or read events from a +pre-recorded .etl file. This functionality enables the module to adapt to +different scenarios, such as real-time event monitoring or analyzing historical +data. + +This input currently supports manifest-based, MOF (classic) and TraceLogging +providers while WPP providers are not supported. +https://learn.microsoft.com/en-us/windows/win32/etw/about-event-tracing#types-of-providers[Here] +you can find more information about the available types of providers. + +It has been tested in the Windows versions supported by {beatname_uc}, starting +from Windows 10 and Windows Server 2016. In addition, administrative privileges +are required to control event tracing sessions. Example configurations: @@ -35,7 +51,7 @@ Read from a provider by name: match_all_keyword: 0 ---- -Same provider can be defined by its GUID: +Read from a provider by its GUID: ["source","yaml",subs="attributes"] ---- {beatname_lc}.inputs: @@ -49,7 +65,7 @@ Same provider can be defined by its GUID: match_all_keyword: 0 ---- -Read from a current session: +Read from an existing session: ["source","yaml",subs="attributes"] ---- {beatname_lc}.inputs: @@ -69,7 +85,9 @@ Read from a .etl file: file: "C\Windows\System32\Winevt\Logs\Logfile.etl" ---- -NOTE: Examples shown above are mutually exclusive, since the options `provider.name`, `provider.guid`, `session` and `file` cannot be present at the same time. Nevertheless, it is a requirement that one of them appears. +NOTE: Examples shown above are mutually exclusive, the options +`provider.name`, `provider.guid`, `session` and `file` cannot be present at the +same time. Nevertheless, it is a requirement that one of them is present. Multiple providers example: ["source","yaml",subs="attributes"] @@ -95,50 +113,90 @@ Multiple providers example: ==== Configuration options -The `ETW` input supports the following configuration options. +The `etw` input supports the following configuration options plus the +<<{beatname_lc}-input-{type}-common-options>> described later. [float] ==== `file` -Specifies the path to an .etl file for reading ETW events. This file format is commonly used for storing ETW event logs. +Specifies the path to an .etl file for reading ETW events. This file format is +commonly used for storing ETW event logs. [float] ==== `provider.guid` -Identifies the GUID of an ETW provider. To see available providers, use the command `logman query providers`. +Identifies the GUID of an ETW provider. To see available providers, use the +command `logman query providers`. [float] ==== `provider.name` -Specifies the name of the ETW provider. Available providers can be listed using `logman query providers`. +Specifies the name of the ETW provider. Available providers can be listed using +`logman query providers`. [float] ==== `session_name` -When specified a provider, a new session is created. It sets the name for a new ETW session associated with the provider. If not provided, the default is the provider ID prefixed with 'Elastic-'. +When specifying a provider, a new session is created. This controls the name for +the new ETW session it will create. If not specified, the session will be named +using the provider ID prefixed by 'Elastic-'. [float] ==== `trace_level` -Defines the filtering level for events based on severity. Valid options include critical, error, warning, informational, and verbose. +Defines the filtering level for events based on severity. Valid options include +critical, error, warning, informational, and verbose. [float] ==== `match_any_keyword` -An 8-byte bitmask used for filtering events from specific provider subcomponents based on keyword matching. Any matching keyword will enable the event to be written. Default value is `0xfffffffffffffffff` so it matches every available keyword. +An 8-byte bitmask used for filtering events from specific provider subcomponents +based on keyword matching. Any matching keyword will enable the event to be +written. Default value is `0xfffffffffffffffff` so it matches every available +keyword. -Run `logman query providers ""` to list the available keywords for a specific provider. +Run `logman query providers ""` to list the available keywords +for a specific provider. [float] ==== `match_all_keyword` -Similar to MatchAnyKeyword, this 8-byte bitmask filters events that match all specified keyword bits. Default value is `0` to let every event pass. +Similar to MatchAnyKeyword, this 8-byte bitmask filters events that match all +specified keyword bits. Default value is `0` to let every event pass. -Run `logman query providers ""` to list the available keywords for a specific provider. +Run `logman query providers ""` to list the available keywords +for a specific provider. [float] ==== `session` -Names an existing ETW session to read from. Existing sessions can be listed using `logman query -ets`. +Names an existing ETW session to read from. Existing sessions can be listed +using `logman query -ets`. + +[id="{beatname_lc}-input-{type}-common-options"] +include::../../../../filebeat/docs/inputs/input-common-options.asciidoc[] + +[float] +=== Metrics + +This input exposes metrics under the <>. +These metrics are exposed under the `/inputs/` path. They can be used to +observe the activity of the input. + +You must assign a unique `id` to the input to expose metrics. + +[options="header"] +|======= +| Metric | Description +| `session` | Name of the ETW session. +| `received_events_total` | Total number of events received. +| `discarded_events_total` | Total number of discarded events. +| `errors_total` | Total number of errors. +| `source_lag_time` | Histogram of the difference between timestamped event's creation and reading. +| `arrival_period` | Histogram of the elapsed time between event notification callbacks. +| `processing_time` | Histogram of the elapsed time between event notification callback and publication to the internal queue. +|======= + +Histogram metrics are aggregated over the previous 1024 events. :type!: diff --git a/x-pack/filebeat/input/etw/input.go b/x-pack/filebeat/input/etw/input.go index b5b331b3c923..021805ebdfaf 100644 --- a/x-pack/filebeat/input/etw/input.go +++ b/x-pack/filebeat/input/etw/input.go @@ -17,11 +17,15 @@ import ( stateless "github.com/elastic/beats/v7/filebeat/input/v2/input-stateless" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/feature" + "github.com/elastic/beats/v7/libbeat/monitoring/inputmon" "github.com/elastic/beats/v7/x-pack/libbeat/reader/etw" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" + "github.com/elastic/elastic-agent-libs/monitoring" + "github.com/elastic/elastic-agent-libs/monitoring/adapter" + "github.com/rcrowley/go-metrics" "golang.org/x/sync/errgroup" "golang.org/x/sys/windows" ) @@ -65,6 +69,7 @@ func (op *realSessionOperator) stopSession(session *etw.Session) error { // etwInput struct holds the configuration and state for the ETW input type etwInput struct { log *logp.Logger + metrics *inputMetrics config config etwSession *etw.Session publisher stateless.Publisher @@ -109,6 +114,8 @@ func (e *etwInput) Run(ctx input.Context, publisher stateless.Publisher) error { } e.etwSession.Callback = e.consumeEvent e.publisher = publisher + e.metrics = newInputMetrics(e.etwSession.Name, ctx.ID) + defer e.metrics.unregister() // Set up logger with session information e.log = ctx.Logger.With("session", e.etwSession.Name) @@ -149,6 +156,7 @@ func (e *etwInput) Run(ctx input.Context, publisher stateless.Publisher) error { e.log.Debug("starting ETW consumer") defer e.log.Debug("stopped ETW consumer") if err = e.operator.startConsumer(e.etwSession); err != nil { + e.metrics.errors.Inc() return fmt.Errorf("failed running ETW consumer: %w", err) } return nil @@ -239,20 +247,34 @@ func convertFileTimeToGoTime(fileTime64 uint64) time.Time { func (e *etwInput) consumeEvent(record *etw.EventRecord) uintptr { if record == nil { e.log.Error("received null event record") + e.metrics.errors.Inc() return 1 } - e.log.Debugf("received event with ID %d and user-data length %d", record.EventHeader.EventDescriptor.Id, record.UserDataLength) + start := time.Now() + defer func() { + elapsed := time.Since(start) + e.metrics.processingTime.Update(elapsed.Nanoseconds()) + }() data, err := etw.GetEventProperties(record) if err != nil { e.log.Errorw("failed to read event properties", "error", err) + e.metrics.errors.Inc() + e.metrics.dropped.Inc() return 1 } evt := buildEvent(data, record.EventHeader, e.etwSession, e.config) e.publisher.Publish(evt) + e.metrics.events.Inc() + e.metrics.sourceLag.Update(start.Sub(evt.Timestamp).Nanoseconds()) + if !e.metrics.lastCallback.IsZero() { + e.metrics.arrivalPeriod.Update(start.Sub(e.metrics.lastCallback).Nanoseconds()) + } + e.metrics.lastCallback = start + return 0 } @@ -260,7 +282,48 @@ func (e *etwInput) consumeEvent(record *etw.EventRecord) uintptr { func (e *etwInput) Close() { if err := e.operator.stopSession(e.etwSession); err != nil { e.log.Error("failed to shutdown ETW session") + e.metrics.errors.Inc() return } e.log.Info("successfully shutdown") } + +// inputMetrics handles event log metric reporting. +type inputMetrics struct { + unregister func() + + lastCallback time.Time + + name *monitoring.String // name of the etw session being read + events *monitoring.Uint // total number of events received + dropped *monitoring.Uint // total number of discarded events + errors *monitoring.Uint // total number of errors + sourceLag metrics.Sample // histogram of the difference between timestamped event's creation and reading + arrivalPeriod metrics.Sample // histogram of the elapsed time between callbacks. + processingTime metrics.Sample // histogram of the elapsed time between event callback receipt and publication. +} + +// newInputMetrics returns an input metric for windows ETW. +// If id is empty, a nil inputMetric is returned. +func newInputMetrics(session, id string) *inputMetrics { + reg, unreg := inputmon.NewInputRegistry(inputName, id, nil) + out := &inputMetrics{ + unregister: unreg, + name: monitoring.NewString(reg, "session"), + events: monitoring.NewUint(reg, "received_events_total"), + dropped: monitoring.NewUint(reg, "discarded_events_total"), + errors: monitoring.NewUint(reg, "errors_total"), + sourceLag: metrics.NewUniformSample(1024), + arrivalPeriod: metrics.NewUniformSample(1024), + processingTime: metrics.NewUniformSample(1024), + } + out.name.Set(session) + _ = adapter.NewGoMetrics(reg, "source_lag_time", adapter.Accept). + Register("histogram", metrics.NewHistogram(out.sourceLag)) + _ = adapter.NewGoMetrics(reg, "arrival_period", adapter.Accept). + Register("histogram", metrics.NewHistogram(out.arrivalPeriod)) + _ = adapter.NewGoMetrics(reg, "processing_time", adapter.Accept). + Register("histogram", metrics.NewHistogram(out.processingTime)) + + return out +} diff --git a/x-pack/filebeat/input/etw/input_test.go b/x-pack/filebeat/input/etw/input_test.go index fd2673278d37..95c9167a6964 100644 --- a/x-pack/filebeat/input/etw/input_test.go +++ b/x-pack/filebeat/input/etw/input_test.go @@ -91,6 +91,7 @@ func Test_RunEtwInput_NewSessionError(t *testing.T) { MatchAllKeyword: 0, }, operator: mockOperator, + metrics: newInputMetrics("", ""), } // Run test @@ -131,6 +132,7 @@ func Test_RunEtwInput_AttachToExistingSessionError(t *testing.T) { MatchAllKeyword: 0, }, operator: mockOperator, + metrics: newInputMetrics("", ""), } // Run test @@ -175,6 +177,7 @@ func Test_RunEtwInput_CreateRealtimeSessionError(t *testing.T) { MatchAllKeyword: 0, }, operator: mockOperator, + metrics: newInputMetrics("", ""), } // Run test @@ -231,6 +234,7 @@ func Test_RunEtwInput_StartConsumerError(t *testing.T) { MatchAllKeyword: 0, }, operator: mockOperator, + metrics: newInputMetrics("", ""), } // Run test @@ -287,6 +291,7 @@ func Test_RunEtwInput_Success(t *testing.T) { MatchAllKeyword: 0, }, operator: mockOperator, + metrics: newInputMetrics("", ""), } // Run test From 07e231bd3842957119630d4b609b7653c34f5223 Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Thu, 7 Mar 2024 09:23:06 -0500 Subject: [PATCH 24/26] [Automation] Bump Golang version to 1.21.8 (#38209) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore: Update stan Dockerfile Made with ❤️️ by updatecli * chore: Update Metricbeat debug Dockerfile Made with ❤️️ by updatecli * chore: Update Auditbeat Dockerfile Made with ❤️️ by updatecli * chore: Update Filebeat debug Dockerfile Made with ❤️️ by updatecli * chore: Update Packetbeat Dockerfile Made with ❤️️ by updatecli * chore: Update Heartbeat Dockerfile Made with ❤️️ by updatecli * chore: Update .golangci.yml Made with ❤️️ by updatecli * chore: Update NATS module Dockerfile Made with ❤️️ by updatecli * chore: Update .go-version Made with ❤️️ by updatecli * chore: Update version.asciidoc Made with ❤️️ by updatecli * chore: Update Metricbeat Dockerfile Made with ❤️️ by updatecli * chore: Update HTTP module Dockerfile Made with ❤️️ by updatecli * chore: Update Heartbeat debug Dockerfile Made with ❤️️ by updatecli * chore: Update from vsphere Dockerfile Made with ❤️️ by updatecli * chore: Update Functionbeat Dockerfile Made with ❤️️ by updatecli * Update changelog. --------- Co-authored-by: apmmachine Co-authored-by: Craig MacKenzie --- .go-version | 2 +- .golangci.yml | 8 ++++---- CHANGELOG.next.asciidoc | 2 +- auditbeat/Dockerfile | 2 +- dev-tools/kubernetes/filebeat/Dockerfile.debug | 2 +- dev-tools/kubernetes/heartbeat/Dockerfile.debug | 2 +- dev-tools/kubernetes/metricbeat/Dockerfile.debug | 2 +- heartbeat/Dockerfile | 2 +- libbeat/docs/version.asciidoc | 2 +- metricbeat/Dockerfile | 2 +- metricbeat/module/http/_meta/Dockerfile | 2 +- metricbeat/module/nats/_meta/Dockerfile | 2 +- metricbeat/module/vsphere/_meta/Dockerfile | 2 +- packetbeat/Dockerfile | 2 +- x-pack/functionbeat/Dockerfile | 2 +- x-pack/metricbeat/module/stan/_meta/Dockerfile | 2 +- 16 files changed, 19 insertions(+), 19 deletions(-) diff --git a/.go-version b/.go-version index 8819d012ceed..428abfd24fbe 100644 --- a/.go-version +++ b/.go-version @@ -1 +1 @@ -1.21.7 +1.21.8 diff --git a/.golangci.yml b/.golangci.yml index 79b77eab0d1a..4fd7b93f1b19 100755 --- a/.golangci.yml +++ b/.golangci.yml @@ -114,7 +114,7 @@ linters-settings: gosimple: # Select the Go version to target. The default is '1.13'. - go: "1.21.7" + go: "1.21.8" nakedret: # make an issue if func has more lines of code than this setting and it has naked returns; default is 30 @@ -132,19 +132,19 @@ linters-settings: staticcheck: # Select the Go version to target. The default is '1.13'. - go: "1.21.7" + go: "1.21.8" checks: ["all"] stylecheck: # Select the Go version to target. The default is '1.13'. - go: "1.21.7" + go: "1.21.8" # Disabled: # ST1005: error strings should not be capitalized checks: ["all", "-ST1005"] unused: # Select the Go version to target. The default is '1.13'. - go: "1.21.7" + go: "1.21.8" gosec: excludes: diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 2c2a762be6ca..faa7f439ce94 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -10,7 +10,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] *Affecting all Beats* -- Upgrade to Go 1.21.7. Removes support for Windows 8.1. See https://tip.golang.org/doc/go1.21#windows. {pull}37913[37913] +- Upgrade to Go 1.21.8. Removes support for Windows 8.1. See https://tip.golang.org/doc/go1.21#windows. {pull}38209[38209] - add_cloud_metadata processor: `huawei` provider is now treated as `openstack`. Huawei cloud runs on OpenStack platform, and when viewed from a metadata API standpoint, it is impossible to differentiate it from OpenStack. If you know that your deployments run on Huawei Cloud exclusively, and you wish to have `cloud.provider` value as `huawei`, diff --git a/auditbeat/Dockerfile b/auditbeat/Dockerfile index df038d2edf83..982680c7f630 100644 --- a/auditbeat/Dockerfile +++ b/auditbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.21.7 +FROM golang:1.21.8 RUN \ apt-get update \ diff --git a/dev-tools/kubernetes/filebeat/Dockerfile.debug b/dev-tools/kubernetes/filebeat/Dockerfile.debug index e8dfaf392aba..b46b2a73cc29 100644 --- a/dev-tools/kubernetes/filebeat/Dockerfile.debug +++ b/dev-tools/kubernetes/filebeat/Dockerfile.debug @@ -1,4 +1,4 @@ -FROM golang:1.21.7 as builder +FROM golang:1.21.8 as builder ENV PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/go/bin:/usr/local/go/bin diff --git a/dev-tools/kubernetes/heartbeat/Dockerfile.debug b/dev-tools/kubernetes/heartbeat/Dockerfile.debug index 473ce7484c83..e1e9dddabacf 100644 --- a/dev-tools/kubernetes/heartbeat/Dockerfile.debug +++ b/dev-tools/kubernetes/heartbeat/Dockerfile.debug @@ -1,4 +1,4 @@ -FROM golang:1.21.7 as builder +FROM golang:1.21.8 as builder ENV PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/go/bin:/usr/local/go/bin diff --git a/dev-tools/kubernetes/metricbeat/Dockerfile.debug b/dev-tools/kubernetes/metricbeat/Dockerfile.debug index 8adf8a45901b..0b81f3635d9d 100644 --- a/dev-tools/kubernetes/metricbeat/Dockerfile.debug +++ b/dev-tools/kubernetes/metricbeat/Dockerfile.debug @@ -1,4 +1,4 @@ -FROM golang:1.21.7 as builder +FROM golang:1.21.8 as builder ENV PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/go/bin:/usr/local/go/bin diff --git a/heartbeat/Dockerfile b/heartbeat/Dockerfile index eb52ad4d1300..7fc0102259c3 100644 --- a/heartbeat/Dockerfile +++ b/heartbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.21.7 +FROM golang:1.21.8 RUN \ apt-get update \ diff --git a/libbeat/docs/version.asciidoc b/libbeat/docs/version.asciidoc index 1cae4fe1ad86..5cb4f7422b51 100644 --- a/libbeat/docs/version.asciidoc +++ b/libbeat/docs/version.asciidoc @@ -1,6 +1,6 @@ :stack-version: 8.13.0 :doc-branch: main -:go-version: 1.21.7 +:go-version: 1.21.8 :release-state: unreleased :python: 3.7 :docker: 1.12 diff --git a/metricbeat/Dockerfile b/metricbeat/Dockerfile index 31f13aeea2c1..b6da04173167 100644 --- a/metricbeat/Dockerfile +++ b/metricbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.21.7 +FROM golang:1.21.8 RUN \ apt update \ diff --git a/metricbeat/module/http/_meta/Dockerfile b/metricbeat/module/http/_meta/Dockerfile index a46a3dbb3e27..e855e12fe030 100644 --- a/metricbeat/module/http/_meta/Dockerfile +++ b/metricbeat/module/http/_meta/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.21.7 +FROM golang:1.21.8 COPY test/main.go main.go diff --git a/metricbeat/module/nats/_meta/Dockerfile b/metricbeat/module/nats/_meta/Dockerfile index 4df0b81a9127..8d06be18187d 100644 --- a/metricbeat/module/nats/_meta/Dockerfile +++ b/metricbeat/module/nats/_meta/Dockerfile @@ -2,7 +2,7 @@ ARG NATS_VERSION=2.0.4 FROM nats:$NATS_VERSION # build stage -FROM golang:1.21.7 AS build-env +FROM golang:1.21.8 AS build-env RUN apt-get install git mercurial gcc RUN git clone https://github.com/nats-io/nats.go.git /nats-go RUN cd /nats-go/examples/nats-bench && git checkout tags/v1.10.0 && go build . diff --git a/metricbeat/module/vsphere/_meta/Dockerfile b/metricbeat/module/vsphere/_meta/Dockerfile index 3db3cccbab27..a5268391f732 100644 --- a/metricbeat/module/vsphere/_meta/Dockerfile +++ b/metricbeat/module/vsphere/_meta/Dockerfile @@ -1,5 +1,5 @@ ARG VSPHERE_GOLANG_VERSION -FROM golang:1.21.7 +FROM golang:1.21.8 RUN apt-get install curl git RUN go install github.com/vmware/govmomi/vcsim@v0.30.4 diff --git a/packetbeat/Dockerfile b/packetbeat/Dockerfile index 17075d9da65b..e565e21af41d 100644 --- a/packetbeat/Dockerfile +++ b/packetbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.21.7 +FROM golang:1.21.8 RUN \ apt-get update \ diff --git a/x-pack/functionbeat/Dockerfile b/x-pack/functionbeat/Dockerfile index b2c858699214..810dd6a9cf16 100644 --- a/x-pack/functionbeat/Dockerfile +++ b/x-pack/functionbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.21.7 +FROM golang:1.21.8 RUN \ apt-get update \ diff --git a/x-pack/metricbeat/module/stan/_meta/Dockerfile b/x-pack/metricbeat/module/stan/_meta/Dockerfile index ffce883a60c3..14cf1ba28317 100644 --- a/x-pack/metricbeat/module/stan/_meta/Dockerfile +++ b/x-pack/metricbeat/module/stan/_meta/Dockerfile @@ -2,7 +2,7 @@ ARG STAN_VERSION=0.15.1 FROM nats-streaming:$STAN_VERSION # build stage -FROM golang:1.21.7 AS build-env +FROM golang:1.21.8 AS build-env RUN apt-get install git mercurial gcc RUN git clone https://github.com/nats-io/stan.go.git /stan-go RUN cd /stan-go/examples/stan-bench && git checkout tags/v0.5.2 && go build . From ca07b8e680c9029648fc401d1a25bffff59a2a91 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Fri, 8 Mar 2024 09:46:36 +0100 Subject: [PATCH 25/26] x-pack/filebeat/input/httpjson: Apply rate limiting to all responses (#38161) - Drain the body before rate limiting. - Apply rate limiting to all responses, waiting immediately. Retry if the response was not successful and there was a rate limit wait (even if immediately expired), otherwise return. - Improve names of variables `epoch` and `activeLimit`. Co-authored-by: Dan Kortschak --- .../filebeat/input/httpjson/rate_limiter.go | 73 +++++++++-------- .../input/httpjson/rate_limiter_test.go | 78 ++++++++++--------- x-pack/filebeat/input/httpjson/request.go | 22 +++--- 3 files changed, 93 insertions(+), 80 deletions(-) diff --git a/x-pack/filebeat/input/httpjson/rate_limiter.go b/x-pack/filebeat/input/httpjson/rate_limiter.go index 30c50ae3f052..d82f5829be8b 100644 --- a/x-pack/filebeat/input/httpjson/rate_limiter.go +++ b/x-pack/filebeat/input/httpjson/rate_limiter.go @@ -42,35 +42,37 @@ func (r *rateLimiter) execute(ctx context.Context, f func() (*http.Response, err for { resp, err := f() if err != nil { - return nil, fmt.Errorf("failed to read http.response.body: %w", err) + return nil, err } - if r == nil || resp.StatusCode == http.StatusOK { + if r == nil { return resp, nil } - if resp.StatusCode != http.StatusTooManyRequests { - return nil, fmt.Errorf("http request was unsuccessful with a status code %d", resp.StatusCode) + applied, err := r.applyRateLimit(ctx, resp) + if err != nil { + return nil, fmt.Errorf("error applying rate limit: %w", err) } - if err := r.applyRateLimit(ctx, resp); err != nil { - return nil, err + if resp.StatusCode == http.StatusOK || !applied { + return resp, nil } } } -// applyRateLimit applies appropriate rate limit if specified in the HTTP Header of the response -func (r *rateLimiter) applyRateLimit(ctx context.Context, resp *http.Response) error { - epoch, err := r.getRateLimit(resp) +// applyRateLimit applies appropriate rate limit if specified in the HTTP Header of the response. +// It returns a bool indicating whether a limit was reached. +func (r *rateLimiter) applyRateLimit(ctx context.Context, resp *http.Response) (bool, error) { + limitReached, resumeAt, err := r.getRateLimit(resp) if err != nil { - return err + return limitReached, err } - t := time.Unix(epoch, 0) + t := time.Unix(resumeAt, 0) w := time.Until(t) - if epoch == 0 || w <= 0 { + if resumeAt == 0 || w <= 0 { r.log.Debugf("Rate Limit: No need to apply rate limit.") - return nil + return limitReached, nil } r.log.Debugf("Rate Limit: Wait until %v for the rate limit to reset.", t) timer := time.NewTimer(w) @@ -80,24 +82,25 @@ func (r *rateLimiter) applyRateLimit(ctx context.Context, resp *http.Response) e <-timer.C } r.log.Info("Context done.") - return nil + return limitReached, nil case <-timer.C: r.log.Debug("Rate Limit: time is up.") - return nil + return limitReached, nil } } // getRateLimit gets the rate limit value if specified in the response, -// and returns an int64 value in seconds since unix epoch for rate limit reset time. +// and returns a bool indicating whether a limit was reached, and +// an int64 value in seconds since unix epoch for rate limit reset time. // When there is a remaining rate limit quota, or when the rate limit reset time has expired, it // returns 0 for the epoch value. -func (r *rateLimiter) getRateLimit(resp *http.Response) (int64, error) { +func (r *rateLimiter) getRateLimit(resp *http.Response) (bool, int64, error) { if r == nil { - return 0, nil + return false, 0, nil } if r.remaining == nil { - return 0, nil + return false, 0, nil } tr := transformable{} @@ -106,16 +109,17 @@ func (r *rateLimiter) getRateLimit(resp *http.Response) (int64, error) { remaining, _ := r.remaining.Execute(ctx, tr, "rate-limit_remaining", nil, r.log) if remaining == "" { - return 0, errors.New("remaining value is empty") + r.log.Infow("get rate limit", "error", errors.New("remaining value is empty")) + return false, 0, nil } m, err := strconv.ParseInt(remaining, 10, 64) if err != nil { - return 0, fmt.Errorf("failed to parse rate-limit remaining value: %w", err) + return false, 0, fmt.Errorf("failed to parse rate-limit remaining value: %w", err) } // by default, httpjson will continue requests until Limit is 0 // can optionally stop requests "early" - var activeLimit int64 = 0 + var minRemaining int64 = 0 if r.earlyLimit != nil { earlyLimit := *r.earlyLimit if earlyLimit > 0 && earlyLimit < 1 { @@ -123,37 +127,38 @@ func (r *rateLimiter) getRateLimit(resp *http.Response) (int64, error) { if limit != "" { l, err := strconv.ParseInt(limit, 10, 64) if err == nil { - activeLimit = l - int64(earlyLimit*float64(l)) + minRemaining = l - int64(earlyLimit*float64(l)) } } } else if earlyLimit >= 1 { - activeLimit = int64(earlyLimit) + minRemaining = int64(earlyLimit) } } - r.log.Debugf("Rate Limit: Using active Early Limit: %f", activeLimit) - if m > activeLimit { - return 0, nil + r.log.Debugf("Rate Limit: Using active Early Limit: %d", minRemaining) + if m > minRemaining { + return false, 0, nil } if r.reset == nil { r.log.Warn("reset rate limit is not set") - return 0, nil + return false, 0, nil } reset, _ := r.reset.Execute(ctx, tr, "rate-limit_reset", nil, r.log) if reset == "" { - return 0, errors.New("reset value is empty") + r.log.Infow("get rate limit", "error", errors.New("reset value is empty")) + return false, 0, nil } - epoch, err := strconv.ParseInt(reset, 10, 64) + resumeAt, err := strconv.ParseInt(reset, 10, 64) if err != nil { - return 0, fmt.Errorf("failed to parse rate-limit reset value: %w", err) + return false, 0, fmt.Errorf("failed to parse rate-limit reset value: %w", err) } - if timeNow().Unix() > epoch { - return 0, nil + if timeNow().Unix() > resumeAt { + return true, 0, nil } - return epoch, nil + return true, resumeAt, nil } diff --git a/x-pack/filebeat/input/httpjson/rate_limiter_test.go b/x-pack/filebeat/input/httpjson/rate_limiter_test.go index fe928eb4f3d9..3fdb73fc44cc 100644 --- a/x-pack/filebeat/input/httpjson/rate_limiter_test.go +++ b/x-pack/filebeat/input/httpjson/rate_limiter_test.go @@ -16,7 +16,7 @@ import ( ) // Test getRateLimit function with a remaining quota, expect to receive 0, nil. -func TestGetRateLimitReturns0IfRemainingQuota(t *testing.T) { +func TestGetRateLimitReturnsFalse0IfRemainingQuota(t *testing.T) { header := make(http.Header) header.Add("X-Rate-Limit-Limit", "120") header.Add("X-Rate-Limit-Remaining", "118") @@ -34,12 +34,13 @@ func TestGetRateLimitReturns0IfRemainingQuota(t *testing.T) { log: logp.NewLogger(""), } resp := &http.Response{Header: header} - epoch, err := rateLimit.getRateLimit(resp) + applied, resumeAt, err := rateLimit.getRateLimit(resp) assert.NoError(t, err) - assert.EqualValues(t, 0, epoch) + assert.False(t, applied) + assert.EqualValues(t, 0, resumeAt) } -func TestGetRateLimitReturns0IfEpochInPast(t *testing.T) { +func TestGetRateLimitReturnsTrue0IfResumeAtInPast(t *testing.T) { header := make(http.Header) header.Add("X-Rate-Limit-Limit", "10") header.Add("X-Rate-Limit-Remaining", "0") @@ -57,20 +58,21 @@ func TestGetRateLimitReturns0IfEpochInPast(t *testing.T) { log: logp.NewLogger(""), } resp := &http.Response{Header: header} - epoch, err := rateLimit.getRateLimit(resp) + applied, resumeAt, err := rateLimit.getRateLimit(resp) assert.NoError(t, err) - assert.EqualValues(t, 0, epoch) + assert.True(t, applied) + assert.EqualValues(t, 0, resumeAt) } func TestGetRateLimitReturnsResetValue(t *testing.T) { - epoch := int64(1604582732 + 100) + reset := int64(1604582732 + 100) timeNow = func() time.Time { return time.Unix(1604582732, 0).UTC() } t.Cleanup(func() { timeNow = time.Now }) header := make(http.Header) header.Add("X-Rate-Limit-Limit", "10") header.Add("X-Rate-Limit-Remaining", "0") - header.Add("X-Rate-Limit-Reset", strconv.FormatInt(epoch, 10)) + header.Add("X-Rate-Limit-Reset", strconv.FormatInt(reset, 10)) tplLimit := &valueTpl{} tplReset := &valueTpl{} tplRemaining := &valueTpl{} @@ -84,22 +86,23 @@ func TestGetRateLimitReturnsResetValue(t *testing.T) { log: logp.NewLogger(""), } resp := &http.Response{Header: header} - epoch2, err := rateLimit.getRateLimit(resp) + applied, resumeAt, err := rateLimit.getRateLimit(resp) assert.NoError(t, err) - assert.EqualValues(t, 1604582832, epoch2) + assert.True(t, applied) + assert.EqualValues(t, reset, resumeAt) } // Test getRateLimit function with a remaining quota, using default early limit -// expect to receive 0, nil. +// expect to receive false, 0, nil. func TestGetRateLimitReturns0IfEarlyLimit0(t *testing.T) { - resetEpoch := int64(1634579974 + 100) + resetAt := int64(1634579974 + 100) timeNow = func() time.Time { return time.Unix(1634579974, 0).UTC() } t.Cleanup(func() { timeNow = time.Now }) header := make(http.Header) header.Add("X-Rate-Limit-Limit", "120") header.Add("X-Rate-Limit-Remaining", "1") - header.Add("X-Rate-Limit-Reset", strconv.FormatInt(resetEpoch, 10)) + header.Add("X-Rate-Limit-Reset", strconv.FormatInt(resetAt, 10)) tplLimit := &valueTpl{} tplReset := &valueTpl{} tplRemaining := &valueTpl{} @@ -115,22 +118,23 @@ func TestGetRateLimitReturns0IfEarlyLimit0(t *testing.T) { earlyLimit: earlyLimit, } resp := &http.Response{Header: header} - epoch, err := rateLimit.getRateLimit(resp) + applied, resumeAt, err := rateLimit.getRateLimit(resp) assert.NoError(t, err) - assert.EqualValues(t, 0, epoch) + assert.False(t, applied) + assert.EqualValues(t, 0, resumeAt) } // Test getRateLimit function with a remaining limit, but early limit -// expect to receive Reset Time +// expect to receive true, Reset Time func TestGetRateLimitReturnsResetValueIfEarlyLimit1(t *testing.T) { - resetEpoch := int64(1634579974 + 100) + resetAt := int64(1634579974 + 100) timeNow = func() time.Time { return time.Unix(1634579974, 0).UTC() } t.Cleanup(func() { timeNow = time.Now }) header := make(http.Header) header.Add("X-Rate-Limit-Limit", "120") header.Add("X-Rate-Limit-Remaining", "1") - header.Add("X-Rate-Limit-Reset", strconv.FormatInt(resetEpoch, 10)) + header.Add("X-Rate-Limit-Reset", strconv.FormatInt(resetAt, 10)) tplLimit := &valueTpl{} tplReset := &valueTpl{} tplRemaining := &valueTpl{} @@ -146,22 +150,23 @@ func TestGetRateLimitReturnsResetValueIfEarlyLimit1(t *testing.T) { earlyLimit: earlyLimit, } resp := &http.Response{Header: header} - epoch, err := rateLimit.getRateLimit(resp) + applied, resumeAt, err := rateLimit.getRateLimit(resp) assert.NoError(t, err) - assert.EqualValues(t, resetEpoch, epoch) + assert.True(t, applied) + assert.EqualValues(t, resetAt, resumeAt) } // Test getRateLimit function with a remaining quota, using 90% early limit -// expect to receive 0, nil. +// expect to receive false, 0, nil. func TestGetRateLimitReturns0IfEarlyLimitPercent(t *testing.T) { - resetEpoch := int64(1634579974 + 100) + resetAt := int64(1634579974 + 100) timeNow = func() time.Time { return time.Unix(1634579974, 0).UTC() } t.Cleanup(func() { timeNow = time.Now }) header := make(http.Header) header.Add("X-Rate-Limit-Limit", "120") header.Add("X-Rate-Limit-Remaining", "13") - header.Add("X-Rate-Limit-Reset", strconv.FormatInt(resetEpoch, 10)) + header.Add("X-Rate-Limit-Reset", strconv.FormatInt(resetAt, 10)) tplLimit := &valueTpl{} tplReset := &valueTpl{} tplRemaining := &valueTpl{} @@ -177,22 +182,23 @@ func TestGetRateLimitReturns0IfEarlyLimitPercent(t *testing.T) { earlyLimit: earlyLimit, } resp := &http.Response{Header: header} - epoch, err := rateLimit.getRateLimit(resp) + applied, resumeAt, err := rateLimit.getRateLimit(resp) assert.NoError(t, err) - assert.EqualValues(t, 0, epoch) + assert.False(t, applied) + assert.EqualValues(t, 0, resumeAt) } // Test getRateLimit function with a remaining limit, but early limit of 90% -// expect to receive Reset Time +// expect to receive true, Reset Time func TestGetRateLimitReturnsResetValueIfEarlyLimitPercent(t *testing.T) { - resetEpoch := int64(1634579974 + 100) + resetAt := int64(1634579974 + 100) timeNow = func() time.Time { return time.Unix(1634579974, 0).UTC() } t.Cleanup(func() { timeNow = time.Now }) header := make(http.Header) header.Add("X-Rate-Limit-Limit", "120") header.Add("X-Rate-Limit-Remaining", "12") - header.Add("X-Rate-Limit-Reset", strconv.FormatInt(resetEpoch, 10)) + header.Add("X-Rate-Limit-Reset", strconv.FormatInt(resetAt, 10)) tplLimit := &valueTpl{} tplReset := &valueTpl{} tplRemaining := &valueTpl{} @@ -208,21 +214,22 @@ func TestGetRateLimitReturnsResetValueIfEarlyLimitPercent(t *testing.T) { earlyLimit: earlyLimit, } resp := &http.Response{Header: header} - epoch, err := rateLimit.getRateLimit(resp) + applied, resumeAt, err := rateLimit.getRateLimit(resp) assert.NoError(t, err) - assert.EqualValues(t, resetEpoch, epoch) + assert.True(t, applied) + assert.EqualValues(t, resetAt, resumeAt) } // Test getRateLimit function when "Limit" header is missing, when using a Percentage early-limit -// expect to receive 0, nil. (default rate-limiting) +// expect to receive false, 0, nil. (default rate-limiting) func TestGetRateLimitWhenMissingLimit(t *testing.T) { - resetEpoch := int64(1634579974 + 100) + reset := int64(1634579974 + 100) timeNow = func() time.Time { return time.Unix(1634579974, 0).UTC() } t.Cleanup(func() { timeNow = time.Now }) header := make(http.Header) header.Add("X-Rate-Limit-Remaining", "1") - header.Add("X-Rate-Limit-Reset", strconv.FormatInt(resetEpoch, 10)) + header.Add("X-Rate-Limit-Reset", strconv.FormatInt(reset, 10)) tplReset := &valueTpl{} tplRemaining := &valueTpl{} earlyLimit := func(i float64) *float64 { return &i }(0.9) @@ -236,7 +243,8 @@ func TestGetRateLimitWhenMissingLimit(t *testing.T) { earlyLimit: earlyLimit, } resp := &http.Response{Header: header} - epoch, err := rateLimit.getRateLimit(resp) + applied, resumeAt, err := rateLimit.getRateLimit(resp) assert.NoError(t, err) - assert.EqualValues(t, 0, epoch) + assert.False(t, applied) + assert.EqualValues(t, 0, resumeAt) } diff --git a/x-pack/filebeat/input/httpjson/request.go b/x-pack/filebeat/input/httpjson/request.go index 5612f2dc6410..9e60d22ac495 100644 --- a/x-pack/filebeat/input/httpjson/request.go +++ b/x-pack/filebeat/input/httpjson/request.go @@ -236,19 +236,17 @@ func (rf *requestFactory) collectResponse(ctx context.Context, trCtx *transformC func (c *httpClient) do(ctx context.Context, req *http.Request) (*http.Response, error) { resp, err := c.limiter.execute(ctx, func() (*http.Response, error) { - return c.client.Do(req) + resp, err := c.client.Do(req) + if err == nil { + // Read the whole resp.Body so we can release the connection. + // This implementation is inspired by httputil.DumpResponse + resp.Body, err = drainBody(resp.Body) + } + return resp, err }) if err != nil { return nil, err } - defer resp.Body.Close() - - // Read the whole resp.Body so we can release the connection. - // This implementation is inspired by httputil.DumpResponse - resp.Body, err = drainBody(resp.Body) - if err != nil { - return nil, fmt.Errorf("failed to read response body: %w", err) - } if resp.StatusCode >= http.StatusBadRequest { body, _ := io.ReadAll(resp.Body) @@ -939,6 +937,8 @@ func cloneResponse(source *http.Response) (*http.Response, error) { // // This function is a modified version of drainBody from the http/httputil package. func drainBody(b io.ReadCloser) (r1 io.ReadCloser, err error) { + defer b.Close() + if b == nil || b == http.NoBody { // No copying needed. Preserve the magic sentinel meaning of NoBody. return http.NoBody, nil @@ -946,10 +946,10 @@ func drainBody(b io.ReadCloser) (r1 io.ReadCloser, err error) { var buf bytes.Buffer if _, err = buf.ReadFrom(b); err != nil { - return b, err + return b, fmt.Errorf("failed to read http.response.body: %w", err) } if err = b.Close(); err != nil { - return b, err + return b, fmt.Errorf("failed to close http.response.body: %w", err) } return io.NopCloser(&buf), nil From 6d8882b5d653176f2355f317754e6967773fabcf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 8 Mar 2024 12:14:42 -0500 Subject: [PATCH 26/26] build(deps): bump github.com/elastic/elastic-agent-system-metrics from 0.9.1 to 0.9.2 (#38229) * build(deps): bump github.com/elastic/elastic-agent-system-metrics Bumps [github.com/elastic/elastic-agent-system-metrics](https://github.com/elastic/elastic-agent-system-metrics) from 0.9.1 to 0.9.2. - [Release notes](https://github.com/elastic/elastic-agent-system-metrics/releases) - [Changelog](https://github.com/elastic/elastic-agent-system-metrics/blob/main/CHANGELOG.md) - [Commits](https://github.com/elastic/elastic-agent-system-metrics/compare/v0.9.1...v0.9.2) --- updated-dependencies: - dependency-name: github.com/elastic/elastic-agent-system-metrics dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Update NOTICE.txt --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- NOTICE.txt | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index c038c7027e39..f1895fdc1efd 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -13041,11 +13041,11 @@ these terms. -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-system-metrics -Version: v0.9.1 +Version: v0.9.2 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-system-metrics@v0.9.1/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-system-metrics@v0.9.2/LICENSE.txt: Apache License Version 2.0, January 2004 diff --git a/go.mod b/go.mod index d087675031e3..0c1f181033bf 100644 --- a/go.mod +++ b/go.mod @@ -204,7 +204,7 @@ require ( github.com/elastic/elastic-agent-autodiscover v0.6.7 github.com/elastic/elastic-agent-libs v0.7.5 github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3 - github.com/elastic/elastic-agent-system-metrics v0.9.1 + github.com/elastic/elastic-agent-system-metrics v0.9.2 github.com/elastic/go-elasticsearch/v8 v8.12.0 github.com/elastic/mito v1.9.0 github.com/elastic/tk-btf v0.1.0 diff --git a/go.sum b/go.sum index 746d5023ae99..c7dea61a7624 100644 --- a/go.sum +++ b/go.sum @@ -673,8 +673,8 @@ github.com/elastic/elastic-agent-libs v0.7.5 h1:4UMqB3BREvhwecYTs/L23oQp1hs/XUkc github.com/elastic/elastic-agent-libs v0.7.5/go.mod h1:pGMj5myawdqu+xE+WKvM5FQzKQ/MonikkWOzoFTJxaU= github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3 h1:sb+25XJn/JcC9/VL8HX4r4QXSUq4uTNzGS2kxOE7u1U= github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3/go.mod h1:rWarFM7qYxJKsi9WcV6ONcFjH/NA3niDNpTxO+8/GVI= -github.com/elastic/elastic-agent-system-metrics v0.9.1 h1:r0ofKHgPpl+W09ie7tzGcCDC0d4NZbQUv37rSgHf4FM= -github.com/elastic/elastic-agent-system-metrics v0.9.1/go.mod h1:9C1UEfj0P687HAzZepHszN6zXA+2tN2Lx3Osvq1zby8= +github.com/elastic/elastic-agent-system-metrics v0.9.2 h1:/tvTKOt55EerU0WwGFoDhBlyWLgxyv7d8xCbny0bciw= +github.com/elastic/elastic-agent-system-metrics v0.9.2/go.mod h1:VfJnKw4Jqrd9ddljXCwaGKJgN+7ADyyGk089NaXVsf0= github.com/elastic/elastic-transport-go/v8 v8.4.0 h1:EKYiH8CHd33BmMna2Bos1rDNMM89+hdgcymI+KzJCGE= github.com/elastic/elastic-transport-go/v8 v8.4.0/go.mod h1:YLHer5cj0csTzNFXoNQ8qhtGY1GTvSqPnKWKaqQE3Hk= github.com/elastic/fsevents v0.0.0-20181029231046-e1d381a4d270 h1:cWPqxlPtir4RoQVCpGSRXmLqjEHpJKbR60rxh1nQZY4=